2026-03-24T02:09:50.605 INFO:root:teuthology version: 1.2.4.dev6+g1c580df7a 2026-03-24T02:09:50.613 DEBUG:teuthology.report:Pushing job info to http://localhost:8080 2026-03-24T02:09:50.644 INFO:teuthology.run:Config: archive_path: /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3539 branch: tentacle description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-journal-stress-workunit} email: null first_in_suite: false flavor: default job_id: '3539' ktype: distro last_in_suite: false machine_type: vps meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run four rbd-mirror daemons per cluster - desc: run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon name: kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps no_nested_subset: false os_type: ubuntu os_version: '22.04' overrides: admin_socket: branch: tentacle ansible.cephlab: branch: main repo: https://github.com/kshtsk/ceph-cm-ansible.git skip_tags: nagios,monitoring-scripts,hostname,pubkeys,zap,sudoers,kerberos,ntp-client,resolvconf,cpan,nfs vars: logical_volumes: lv_1: scratch_dev: true size: 25%VG vg: vg_nvme lv_2: scratch_dev: true size: 25%VG vg: vg_nvme lv_3: scratch_dev: true size: 25%VG vg: vg_nvme lv_4: scratch_dev: true size: 25%VG vg: vg_nvme timezone: UTC volume_groups: vg_nvme: pvs: /dev/vdb,/dev/vdc,/dev/vdd,/dev/vde ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 rbd mirror image policy type: simple client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bluefs allocator: hybrid bluestore allocator: hybrid bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd mclock iops capacity threshold hdd: 49000 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: 70f8415b300f041766fa27faf7d5472699e32388 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs cephadm: cephadm_binary_url: https://download.ceph.com/rpm-20.2.0/el9/noarch/cephadm install: ceph: flavor: default sha1: 70f8415b300f041766fa27faf7d5472699e32388 extra_system_packages: deb: - python3-jmespath - python3-xmltodict - s3cmd rpm: - bzip2 - perl-Test-Harness - python3-jmespath - python3-xmltodict - s3cmd thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: tt-tentacle sha1: 0392f78529848ec72469e8e431875cb98d3a5fb4 owner: kyr priority: 1000 repo: https://github.com/ceph/ceph.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 3051 sha1: 70f8415b300f041766fa27faf7d5472699e32388 sleep_before_teardown: 0 subset: 1/128 suite: rbd suite_branch: tt-tentacle suite_path: /home/teuthos/src/github.com_kshtsk_ceph_0392f78529848ec72469e8e431875cb98d3a5fb4/qa suite_relpath: qa suite_repo: https://github.com/kshtsk/ceph.git suite_sha1: 0392f78529848ec72469e8e431875cb98d3a5fb4 targets: vm06.local: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFMdzVQ4qWHGmJfQDRxmfOu8MjX2Enb0f1yGF9w/uaSFB9MTnpB4aag7G6CcWBNqpWlI9EJwSapJNyvZr8HZKHc= vm07.local: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK5C8C/vVG/hbJJiioODjMgoQKxU7Psh6pK8go4Z7ooenLW0HVrniuzl2OdiCD1ftad+WLA/q04SzvgeWCQAHV4= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - rbd-mirror: client: cluster1.client.mirror.0 thrash: true - rbd-mirror: client: cluster1.client.mirror.1 thrash: true - rbd-mirror: client: cluster1.client.mirror.2 thrash: true - rbd-mirror: client: cluster1.client.mirror.3 thrash: true - rbd-mirror: client: cluster2.client.mirror.0 thrash: true - rbd-mirror: client: cluster2.client.mirror.1 thrash: true - rbd-mirror: client: cluster2.client.mirror.2 thrash: true - rbd-mirror: client: cluster2.client.mirror.3 thrash: true - rbd-mirror-thrash: cluster: cluster1 - rbd-mirror-thrash: cluster: cluster2 - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror_stress.sh env: CEPH_ARGS: '' MIRROR_POOL_MODE: pool RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_MODE: journal RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_USE_RBD_MIRROR: '1' timeout: 6h teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: clyso-debian-13 teuthology_repo: https://github.com/clyso/teuthology teuthology_sha1: 1c580df7a9c7c2aadc272da296344fd99f27c444 timestamp: 2026-03-20_22:04:26 tube: vps user: kyr verbose: false worker_log: /home/teuthos/.teuthology/dispatcher/dispatcher.vps.2366871 2026-03-24T02:09:50.644 INFO:teuthology.run:suite_path is set to /home/teuthos/src/github.com_kshtsk_ceph_0392f78529848ec72469e8e431875cb98d3a5fb4/qa; will attempt to use it 2026-03-24T02:09:50.644 INFO:teuthology.run:Found tasks at /home/teuthos/src/github.com_kshtsk_ceph_0392f78529848ec72469e8e431875cb98d3a5fb4/qa/tasks 2026-03-24T02:09:50.644 INFO:teuthology.run_tasks:Running task internal.check_packages... 2026-03-24T02:09:50.645 INFO:teuthology.task.internal:Checking packages... 2026-03-24T02:09:50.645 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash '70f8415b300f041766fa27faf7d5472699e32388' 2026-03-24T02:09:50.645 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2026-03-24T02:09:50.645 INFO:teuthology.packaging:ref: None 2026-03-24T02:09:50.645 INFO:teuthology.packaging:tag: None 2026-03-24T02:09:50.645 INFO:teuthology.packaging:branch: tentacle 2026-03-24T02:09:50.645 INFO:teuthology.packaging:sha1: 70f8415b300f041766fa27faf7d5472699e32388 2026-03-24T02:09:50.645 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=tentacle 2026-03-24T02:09:51.427 INFO:teuthology.task.internal:Found packages for ceph version 20.2.0-714-g147f7c6a-1jammy 2026-03-24T02:09:51.428 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2026-03-24T02:09:51.429 INFO:teuthology.task.internal:no buildpackages task found 2026-03-24T02:09:51.429 INFO:teuthology.run_tasks:Running task internal.save_config... 2026-03-24T02:09:51.430 INFO:teuthology.task.internal:Saving configuration 2026-03-24T02:09:51.437 INFO:teuthology.run_tasks:Running task internal.check_lock... 2026-03-24T02:09:51.438 INFO:teuthology.task.internal.check_lock:Checking locks... 2026-03-24T02:09:51.446 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'vm06.local', 'description': '/archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3539', 'up': True, 'machine_type': 'vps', 'is_vm': True, 'vm_host': {'name': 'localhost', 'description': None, 'up': True, 'machine_type': 'libvirt', 'is_vm': False, 'vm_host': None, 'os_type': None, 'os_version': None, 'arch': None, 'locked': True, 'locked_since': None, 'locked_by': None, 'mac_address': None, 'ssh_pub_key': None}, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-03-24 02:08:35.309033', 'locked_by': 'kyr', 'mac_address': '52:55:00:00:00:06', 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFMdzVQ4qWHGmJfQDRxmfOu8MjX2Enb0f1yGF9w/uaSFB9MTnpB4aag7G6CcWBNqpWlI9EJwSapJNyvZr8HZKHc='} 2026-03-24T02:09:51.452 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'vm07.local', 'description': '/archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3539', 'up': True, 'machine_type': 'vps', 'is_vm': True, 'vm_host': {'name': 'localhost', 'description': None, 'up': True, 'machine_type': 'libvirt', 'is_vm': False, 'vm_host': None, 'os_type': None, 'os_version': None, 'arch': None, 'locked': True, 'locked_since': None, 'locked_by': None, 'mac_address': None, 'ssh_pub_key': None}, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-03-24 02:08:35.308609', 'locked_by': 'kyr', 'mac_address': '52:55:00:00:00:07', 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK5C8C/vVG/hbJJiioODjMgoQKxU7Psh6pK8go4Z7ooenLW0HVrniuzl2OdiCD1ftad+WLA/q04SzvgeWCQAHV4='} 2026-03-24T02:09:51.452 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2026-03-24T02:09:51.453 INFO:teuthology.task.internal:roles: ubuntu@vm06.local - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2026-03-24T02:09:51.453 INFO:teuthology.task.internal:roles: ubuntu@vm07.local - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2026-03-24T02:09:51.453 INFO:teuthology.run_tasks:Running task console_log... 2026-03-24T02:09:51.460 DEBUG:teuthology.task.console_log:vm06 does not support IPMI; excluding 2026-03-24T02:09:51.466 DEBUG:teuthology.task.console_log:vm07 does not support IPMI; excluding 2026-03-24T02:09:51.466 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fe9f21a4ca0>, signals=[15]) 2026-03-24T02:09:51.466 INFO:teuthology.run_tasks:Running task internal.connect... 2026-03-24T02:09:51.467 INFO:teuthology.task.internal:Opening connections... 2026-03-24T02:09:51.467 DEBUG:teuthology.task.internal:connecting to ubuntu@vm06.local 2026-03-24T02:09:51.468 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm06.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-24T02:09:51.528 DEBUG:teuthology.task.internal:connecting to ubuntu@vm07.local 2026-03-24T02:09:51.529 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm07.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-24T02:09:51.588 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2026-03-24T02:09:51.590 DEBUG:teuthology.orchestra.run.vm06:> uname -m 2026-03-24T02:09:51.611 INFO:teuthology.orchestra.run.vm06.stdout:x86_64 2026-03-24T02:09:51.612 DEBUG:teuthology.orchestra.run.vm06:> cat /etc/os-release 2026-03-24T02:09:51.655 INFO:teuthology.orchestra.run.vm06.stdout:PRETTY_NAME="Ubuntu 22.04.5 LTS" 2026-03-24T02:09:51.656 INFO:teuthology.orchestra.run.vm06.stdout:NAME="Ubuntu" 2026-03-24T02:09:51.656 INFO:teuthology.orchestra.run.vm06.stdout:VERSION_ID="22.04" 2026-03-24T02:09:51.656 INFO:teuthology.orchestra.run.vm06.stdout:VERSION="22.04.5 LTS (Jammy Jellyfish)" 2026-03-24T02:09:51.656 INFO:teuthology.orchestra.run.vm06.stdout:VERSION_CODENAME=jammy 2026-03-24T02:09:51.656 INFO:teuthology.orchestra.run.vm06.stdout:ID=ubuntu 2026-03-24T02:09:51.656 INFO:teuthology.orchestra.run.vm06.stdout:ID_LIKE=debian 2026-03-24T02:09:51.656 INFO:teuthology.orchestra.run.vm06.stdout:HOME_URL="https://www.ubuntu.com/" 2026-03-24T02:09:51.656 INFO:teuthology.orchestra.run.vm06.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2026-03-24T02:09:51.656 INFO:teuthology.orchestra.run.vm06.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2026-03-24T02:09:51.656 INFO:teuthology.orchestra.run.vm06.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2026-03-24T02:09:51.656 INFO:teuthology.orchestra.run.vm06.stdout:UBUNTU_CODENAME=jammy 2026-03-24T02:09:51.656 INFO:teuthology.lock.ops:Updating vm06.local on lock server 2026-03-24T02:09:51.662 DEBUG:teuthology.orchestra.run.vm07:> uname -m 2026-03-24T02:09:51.674 INFO:teuthology.orchestra.run.vm07.stdout:x86_64 2026-03-24T02:09:51.674 DEBUG:teuthology.orchestra.run.vm07:> cat /etc/os-release 2026-03-24T02:09:51.724 INFO:teuthology.orchestra.run.vm07.stdout:PRETTY_NAME="Ubuntu 22.04.5 LTS" 2026-03-24T02:09:51.724 INFO:teuthology.orchestra.run.vm07.stdout:NAME="Ubuntu" 2026-03-24T02:09:51.725 INFO:teuthology.orchestra.run.vm07.stdout:VERSION_ID="22.04" 2026-03-24T02:09:51.725 INFO:teuthology.orchestra.run.vm07.stdout:VERSION="22.04.5 LTS (Jammy Jellyfish)" 2026-03-24T02:09:51.725 INFO:teuthology.orchestra.run.vm07.stdout:VERSION_CODENAME=jammy 2026-03-24T02:09:51.725 INFO:teuthology.orchestra.run.vm07.stdout:ID=ubuntu 2026-03-24T02:09:51.725 INFO:teuthology.orchestra.run.vm07.stdout:ID_LIKE=debian 2026-03-24T02:09:51.725 INFO:teuthology.orchestra.run.vm07.stdout:HOME_URL="https://www.ubuntu.com/" 2026-03-24T02:09:51.725 INFO:teuthology.orchestra.run.vm07.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2026-03-24T02:09:51.725 INFO:teuthology.orchestra.run.vm07.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2026-03-24T02:09:51.725 INFO:teuthology.orchestra.run.vm07.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2026-03-24T02:09:51.725 INFO:teuthology.orchestra.run.vm07.stdout:UBUNTU_CODENAME=jammy 2026-03-24T02:09:51.725 INFO:teuthology.lock.ops:Updating vm07.local on lock server 2026-03-24T02:09:51.730 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2026-03-24T02:09:51.732 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2026-03-24T02:09:51.733 INFO:teuthology.task.internal:Checking for old test directory... 2026-03-24T02:09:51.733 DEBUG:teuthology.orchestra.run.vm06:> test '!' -e /home/ubuntu/cephtest 2026-03-24T02:09:51.735 DEBUG:teuthology.orchestra.run.vm07:> test '!' -e /home/ubuntu/cephtest 2026-03-24T02:09:51.768 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2026-03-24T02:09:51.769 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2026-03-24T02:09:51.769 DEBUG:teuthology.orchestra.run.vm06:> test -z $(ls -A /var/lib/ceph) 2026-03-24T02:09:51.781 DEBUG:teuthology.orchestra.run.vm07:> test -z $(ls -A /var/lib/ceph) 2026-03-24T02:09:51.784 INFO:teuthology.orchestra.run.vm06.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-03-24T02:09:51.813 INFO:teuthology.orchestra.run.vm07.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-03-24T02:09:51.813 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2026-03-24T02:09:51.822 DEBUG:teuthology.orchestra.run.vm06:> test -e /ceph-qa-ready 2026-03-24T02:09:51.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-24T02:09:52.133 DEBUG:teuthology.orchestra.run.vm07:> test -e /ceph-qa-ready 2026-03-24T02:09:52.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-24T02:09:52.368 INFO:teuthology.run_tasks:Running task internal.base... 2026-03-24T02:09:52.370 INFO:teuthology.task.internal:Creating test directory... 2026-03-24T02:09:52.370 DEBUG:teuthology.orchestra.run.vm06:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-03-24T02:09:52.371 DEBUG:teuthology.orchestra.run.vm07:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-03-24T02:09:52.375 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2026-03-24T02:09:52.376 INFO:teuthology.run_tasks:Running task internal.archive... 2026-03-24T02:09:52.377 INFO:teuthology.task.internal:Creating archive directory... 2026-03-24T02:09:52.377 DEBUG:teuthology.orchestra.run.vm06:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-03-24T02:09:52.417 DEBUG:teuthology.orchestra.run.vm07:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-03-24T02:09:52.428 INFO:teuthology.run_tasks:Running task internal.coredump... 2026-03-24T02:09:52.429 INFO:teuthology.task.internal:Enabling coredump saving... 2026-03-24T02:09:52.429 DEBUG:teuthology.orchestra.run.vm06:> test -f /run/.containerenv -o -f /.dockerenv 2026-03-24T02:09:52.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-24T02:09:52.463 DEBUG:teuthology.orchestra.run.vm07:> test -f /run/.containerenv -o -f /.dockerenv 2026-03-24T02:09:52.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-24T02:09:52.468 DEBUG:teuthology.orchestra.run.vm06:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-03-24T02:09:52.505 DEBUG:teuthology.orchestra.run.vm07:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-03-24T02:09:52.515 INFO:teuthology.orchestra.run.vm06.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-24T02:09:52.519 INFO:teuthology.orchestra.run.vm07.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-24T02:09:52.520 INFO:teuthology.orchestra.run.vm06.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-24T02:09:52.523 INFO:teuthology.orchestra.run.vm07.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-24T02:09:52.524 INFO:teuthology.run_tasks:Running task internal.sudo... 2026-03-24T02:09:52.526 INFO:teuthology.task.internal:Configuring sudo... 2026-03-24T02:09:52.526 DEBUG:teuthology.orchestra.run.vm06:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-03-24T02:09:52.565 DEBUG:teuthology.orchestra.run.vm07:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-03-24T02:09:52.573 INFO:teuthology.run_tasks:Running task internal.syslog... 2026-03-24T02:09:52.576 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2026-03-24T02:09:52.576 DEBUG:teuthology.orchestra.run.vm06:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-03-24T02:09:52.617 DEBUG:teuthology.orchestra.run.vm07:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-03-24T02:09:52.620 DEBUG:teuthology.orchestra.run.vm06:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-03-24T02:09:52.663 DEBUG:teuthology.orchestra.run.vm06:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-03-24T02:09:52.708 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:09:52.708 DEBUG:teuthology.orchestra.run.vm06:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-03-24T02:09:52.760 DEBUG:teuthology.orchestra.run.vm07:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-03-24T02:09:52.764 DEBUG:teuthology.orchestra.run.vm07:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-03-24T02:09:52.808 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:09:52.808 DEBUG:teuthology.orchestra.run.vm07:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-03-24T02:09:52.859 DEBUG:teuthology.orchestra.run.vm06:> sudo service rsyslog restart 2026-03-24T02:09:52.860 DEBUG:teuthology.orchestra.run.vm07:> sudo service rsyslog restart 2026-03-24T02:09:52.918 INFO:teuthology.run_tasks:Running task internal.timer... 2026-03-24T02:09:52.920 INFO:teuthology.task.internal:Starting timer... 2026-03-24T02:09:52.920 INFO:teuthology.run_tasks:Running task pcp... 2026-03-24T02:09:52.923 INFO:teuthology.run_tasks:Running task selinux... 2026-03-24T02:09:52.925 INFO:teuthology.task.selinux:Excluding vm06: VMs are not yet supported 2026-03-24T02:09:52.925 INFO:teuthology.task.selinux:Excluding vm07: VMs are not yet supported 2026-03-24T02:09:52.925 DEBUG:teuthology.task.selinux:Getting current SELinux state 2026-03-24T02:09:52.926 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2026-03-24T02:09:52.926 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2026-03-24T02:09:52.926 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2026-03-24T02:09:52.927 DEBUG:teuthology.task:Applying overrides for task ansible.cephlab: {'branch': 'main', 'repo': 'https://github.com/kshtsk/ceph-cm-ansible.git', 'skip_tags': 'nagios,monitoring-scripts,hostname,pubkeys,zap,sudoers,kerberos,ntp-client,resolvconf,cpan,nfs', 'vars': {'logical_volumes': {'lv_1': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_2': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_3': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_4': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}}, 'timezone': 'UTC', 'volume_groups': {'vg_nvme': {'pvs': '/dev/vdb,/dev/vdc,/dev/vdd,/dev/vde'}}}} 2026-03-24T02:09:52.927 DEBUG:teuthology.repo_utils:Setting repo remote to https://github.com/kshtsk/ceph-cm-ansible.git 2026-03-24T02:09:52.929 INFO:teuthology.repo_utils:Fetching github.com_kshtsk_ceph-cm-ansible_main from origin 2026-03-24T02:09:53.430 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthos/src/github.com_kshtsk_ceph-cm-ansible_main to origin/main 2026-03-24T02:09:53.436 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2026-03-24T02:09:53.437 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu", "logical_volumes": {"lv_1": {"scratch_dev": true, "size": "25%VG", "vg": "vg_nvme"}, "lv_2": {"scratch_dev": true, "size": "25%VG", "vg": "vg_nvme"}, "lv_3": {"scratch_dev": true, "size": "25%VG", "vg": "vg_nvme"}, "lv_4": {"scratch_dev": true, "size": "25%VG", "vg": "vg_nvme"}}, "timezone": "UTC", "volume_groups": {"vg_nvme": {"pvs": "/dev/vdb,/dev/vdc,/dev/vdd,/dev/vde"}}}' -i /tmp/teuth_ansible_inventoryisp1j9vs --limit vm06.local,vm07.local /home/teuthos/src/github.com_kshtsk_ceph-cm-ansible_main/cephlab.yml --skip-tags nagios,monitoring-scripts,hostname,pubkeys,zap,sudoers,kerberos,ntp-client,resolvconf,cpan,nfs 2026-03-24T02:12:19.466 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@vm06.local'), Remote(name='ubuntu@vm07.local')] 2026-03-24T02:12:19.467 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@vm06.local' 2026-03-24T02:12:19.467 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm06.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-24T02:12:19.523 DEBUG:teuthology.orchestra.run.vm06:> true 2026-03-24T02:12:19.760 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@vm06.local' 2026-03-24T02:12:19.761 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@vm07.local' 2026-03-24T02:12:19.761 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm07.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-24T02:12:19.822 DEBUG:teuthology.orchestra.run.vm07:> true 2026-03-24T02:12:20.045 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@vm07.local' 2026-03-24T02:12:20.045 INFO:teuthology.run_tasks:Running task clock... 2026-03-24T02:12:20.048 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2026-03-24T02:12:20.048 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-03-24T02:12:20.048 DEBUG:teuthology.orchestra.run.vm06:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-03-24T02:12:20.049 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-03-24T02:12:20.049 DEBUG:teuthology.orchestra.run.vm07:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-03-24T02:12:20.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2026-03-24T02:12:20.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: Command line: ntpd -gq 2026-03-24T02:12:20.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: ---------------------------------------------------- 2026-03-24T02:12:20.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: ntp-4 is maintained by Network Time Foundation, 2026-03-24T02:12:20.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2026-03-24T02:12:20.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: corporation. Support and training for ntp-4 are 2026-03-24T02:12:20.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: available at https://www.nwtime.org/support 2026-03-24T02:12:20.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: ---------------------------------------------------- 2026-03-24T02:12:20.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: proto: precision = 0.031 usec (-25) 2026-03-24T02:12:20.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: basedate set to 2022-02-04 2026-03-24T02:12:20.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: gps base set to 2022-02-06 (week 2196) 2026-03-24T02:12:20.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: leapsecond file ('/usr/share/zoneinfo/leap-seconds.list'): good hash signature 2026-03-24T02:12:20.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: leapsecond file ('/usr/share/zoneinfo/leap-seconds.list'): loaded, expire=2025-12-28T00:00:00Z last=2017-01-01T00:00:00Z ofs=37 2026-03-24T02:12:20.065 INFO:teuthology.orchestra.run.vm06.stderr:24 Mar 02:12:20 ntpd[16255]: leapsecond file ('/usr/share/zoneinfo/leap-seconds.list'): expired 87 days ago 2026-03-24T02:12:20.065 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: Listen and drop on 0 v6wildcard [::]:123 2026-03-24T02:12:20.065 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2026-03-24T02:12:20.066 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: Listen normally on 2 lo 127.0.0.1:123 2026-03-24T02:12:20.066 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: Listen normally on 3 ens3 192.168.123.106:123 2026-03-24T02:12:20.066 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: Listen normally on 4 lo [::1]:123 2026-03-24T02:12:20.066 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: Listen normally on 5 ens3 [fe80::5055:ff:fe00:6%2]:123 2026-03-24T02:12:20.066 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:20 ntpd[16255]: Listening on routing socket on fd #22 for interface updates 2026-03-24T02:12:20.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2026-03-24T02:12:20.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: Command line: ntpd -gq 2026-03-24T02:12:20.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: ---------------------------------------------------- 2026-03-24T02:12:20.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: ntp-4 is maintained by Network Time Foundation, 2026-03-24T02:12:20.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2026-03-24T02:12:20.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: corporation. Support and training for ntp-4 are 2026-03-24T02:12:20.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: available at https://www.nwtime.org/support 2026-03-24T02:12:20.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: ---------------------------------------------------- 2026-03-24T02:12:20.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: proto: precision = 0.029 usec (-25) 2026-03-24T02:12:20.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: basedate set to 2022-02-04 2026-03-24T02:12:20.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: gps base set to 2022-02-06 (week 2196) 2026-03-24T02:12:20.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: leapsecond file ('/usr/share/zoneinfo/leap-seconds.list'): good hash signature 2026-03-24T02:12:20.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: leapsecond file ('/usr/share/zoneinfo/leap-seconds.list'): loaded, expire=2025-12-28T00:00:00Z last=2017-01-01T00:00:00Z ofs=37 2026-03-24T02:12:20.102 INFO:teuthology.orchestra.run.vm07.stderr:24 Mar 02:12:20 ntpd[16237]: leapsecond file ('/usr/share/zoneinfo/leap-seconds.list'): expired 87 days ago 2026-03-24T02:12:20.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: Listen and drop on 0 v6wildcard [::]:123 2026-03-24T02:12:20.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2026-03-24T02:12:20.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: Listen normally on 2 lo 127.0.0.1:123 2026-03-24T02:12:20.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: Listen normally on 3 ens3 192.168.123.107:123 2026-03-24T02:12:20.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: Listen normally on 4 lo [::1]:123 2026-03-24T02:12:20.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: Listen normally on 5 ens3 [fe80::5055:ff:fe00:7%2]:123 2026-03-24T02:12:20.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:20 ntpd[16237]: Listening on routing socket on fd #22 for interface updates 2026-03-24T02:12:21.065 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:21 ntpd[16255]: Soliciting pool server 148.251.54.81 2026-03-24T02:12:21.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:21 ntpd[16237]: Soliciting pool server 195.201.125.53 2026-03-24T02:12:22.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:22 ntpd[16255]: Soliciting pool server 31.209.85.243 2026-03-24T02:12:22.065 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:22 ntpd[16255]: Soliciting pool server 193.141.27.6 2026-03-24T02:12:22.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:22 ntpd[16237]: Soliciting pool server 148.251.54.81 2026-03-24T02:12:22.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:22 ntpd[16237]: Soliciting pool server 139.162.156.95 2026-03-24T02:12:23.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:23 ntpd[16255]: Soliciting pool server 172.236.195.26 2026-03-24T02:12:23.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:23 ntpd[16255]: Soliciting pool server 162.159.200.123 2026-03-24T02:12:23.065 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:23 ntpd[16255]: Soliciting pool server 148.251.5.46 2026-03-24T02:12:23.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:23 ntpd[16237]: Soliciting pool server 193.141.27.6 2026-03-24T02:12:23.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:23 ntpd[16237]: Soliciting pool server 31.209.85.243 2026-03-24T02:12:23.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:23 ntpd[16237]: Soliciting pool server 141.144.230.32 2026-03-24T02:12:24.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:24 ntpd[16255]: Soliciting pool server 172.104.149.161 2026-03-24T02:12:24.065 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:24 ntpd[16255]: Soliciting pool server 212.18.3.18 2026-03-24T02:12:24.065 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:24 ntpd[16255]: Soliciting pool server 195.201.125.53 2026-03-24T02:12:24.065 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:24 ntpd[16255]: Soliciting pool server 85.215.122.93 2026-03-24T02:12:24.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:24 ntpd[16237]: Soliciting pool server 148.251.5.46 2026-03-24T02:12:24.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:24 ntpd[16237]: Soliciting pool server 162.159.200.123 2026-03-24T02:12:24.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:24 ntpd[16237]: Soliciting pool server 77.42.16.222 2026-03-24T02:12:25.065 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:25 ntpd[16255]: Soliciting pool server 129.70.132.34 2026-03-24T02:12:25.065 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:25 ntpd[16255]: Soliciting pool server 5.45.97.204 2026-03-24T02:12:25.065 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:25 ntpd[16255]: Soliciting pool server 139.162.156.95 2026-03-24T02:12:25.065 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:25 ntpd[16255]: Soliciting pool server 91.189.91.157 2026-03-24T02:12:25.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:25 ntpd[16237]: Soliciting pool server 85.215.122.93 2026-03-24T02:12:25.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:25 ntpd[16237]: Soliciting pool server 172.104.149.161 2026-03-24T02:12:25.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:25 ntpd[16237]: Soliciting pool server 185.125.190.57 2026-03-24T02:12:26.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:26 ntpd[16255]: Soliciting pool server 185.125.190.56 2026-03-24T02:12:26.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:26 ntpd[16255]: Soliciting pool server 141.84.43.74 2026-03-24T02:12:26.064 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:26 ntpd[16255]: Soliciting pool server 141.144.230.32 2026-03-24T02:12:26.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:26 ntpd[16237]: Soliciting pool server 91.189.91.157 2026-03-24T02:12:26.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:26 ntpd[16237]: Soliciting pool server 129.70.132.34 2026-03-24T02:12:26.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:26 ntpd[16237]: Soliciting pool server 5.45.97.204 2026-03-24T02:12:27.102 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:27 ntpd[16237]: Soliciting pool server 185.125.190.56 2026-03-24T02:12:27.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:27 ntpd[16237]: Soliciting pool server 141.84.43.74 2026-03-24T02:12:27.103 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:27 ntpd[16237]: Soliciting pool server 2a01:4f8:2b01:101::2 2026-03-24T02:12:28.089 INFO:teuthology.orchestra.run.vm06.stdout:24 Mar 02:12:28 ntpd[16255]: ntpd: time slew +0.000112 s 2026-03-24T02:12:28.090 INFO:teuthology.orchestra.run.vm06.stdout:ntpd: time slew +0.000112s 2026-03-24T02:12:28.111 INFO:teuthology.orchestra.run.vm06.stdout: remote refid st t when poll reach delay offset jitter 2026-03-24T02:12:28.111 INFO:teuthology.orchestra.run.vm06.stdout:============================================================================== 2026-03-24T02:12:28.111 INFO:teuthology.orchestra.run.vm06.stdout: 0.ubuntu.pool.n .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:12:28.111 INFO:teuthology.orchestra.run.vm06.stdout: 1.ubuntu.pool.n .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:12:28.111 INFO:teuthology.orchestra.run.vm06.stdout: 2.ubuntu.pool.n .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:12:28.111 INFO:teuthology.orchestra.run.vm06.stdout: 3.ubuntu.pool.n .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:12:28.111 INFO:teuthology.orchestra.run.vm06.stdout: ntp.ubuntu.com .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:12:29.125 INFO:teuthology.orchestra.run.vm07.stdout:24 Mar 02:12:29 ntpd[16237]: ntpd: time slew -0.004148 s 2026-03-24T02:12:29.125 INFO:teuthology.orchestra.run.vm07.stdout:ntpd: time slew -0.004148s 2026-03-24T02:12:29.146 INFO:teuthology.orchestra.run.vm07.stdout: remote refid st t when poll reach delay offset jitter 2026-03-24T02:12:29.146 INFO:teuthology.orchestra.run.vm07.stdout:============================================================================== 2026-03-24T02:12:29.146 INFO:teuthology.orchestra.run.vm07.stdout: 0.ubuntu.pool.n .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:12:29.146 INFO:teuthology.orchestra.run.vm07.stdout: 1.ubuntu.pool.n .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:12:29.146 INFO:teuthology.orchestra.run.vm07.stdout: 2.ubuntu.pool.n .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:12:29.146 INFO:teuthology.orchestra.run.vm07.stdout: 3.ubuntu.pool.n .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:12:29.146 INFO:teuthology.orchestra.run.vm07.stdout: ntp.ubuntu.com .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:12:29.146 INFO:teuthology.run_tasks:Running task install... 2026-03-24T02:12:29.148 DEBUG:teuthology.task.install:project ceph 2026-03-24T02:12:29.148 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}, 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}} 2026-03-24T02:12:29.148 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}} 2026-03-24T02:12:29.148 INFO:teuthology.task.install:Using flavor: default 2026-03-24T02:12:29.150 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2026-03-24T02:12:29.150 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2026-03-24T02:12:29.150 DEBUG:teuthology.orchestra.run.vm06:> sudo apt-key list | grep Ceph 2026-03-24T02:12:29.150 DEBUG:teuthology.orchestra.run.vm07:> sudo apt-key list | grep Ceph 2026-03-24T02:12:29.190 INFO:teuthology.orchestra.run.vm06.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2026-03-24T02:12:29.211 INFO:teuthology.orchestra.run.vm06.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2026-03-24T02:12:29.211 INFO:teuthology.orchestra.run.vm06.stdout:uid [ unknown] Ceph.com (release key) 2026-03-24T02:12:29.211 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on remote deb x86_64 2026-03-24T02:12:29.211 INFO:teuthology.task.install.deb:Installing system (non-project) packages: python3-jmespath, python3-xmltodict, s3cmd on remote deb x86_64 2026-03-24T02:12:29.211 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=70f8415b300f041766fa27faf7d5472699e32388 2026-03-24T02:12:29.224 INFO:teuthology.orchestra.run.vm07.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2026-03-24T02:12:29.245 INFO:teuthology.orchestra.run.vm07.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2026-03-24T02:12:29.245 INFO:teuthology.orchestra.run.vm07.stdout:uid [ unknown] Ceph.com (release key) 2026-03-24T02:12:29.245 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on remote deb x86_64 2026-03-24T02:12:29.245 INFO:teuthology.task.install.deb:Installing system (non-project) packages: python3-jmespath, python3-xmltodict, s3cmd on remote deb x86_64 2026-03-24T02:12:29.245 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=70f8415b300f041766fa27faf7d5472699e32388 2026-03-24T02:12:29.827 INFO:teuthology.task.install.deb:Pulling from https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default/ 2026-03-24T02:12:29.827 INFO:teuthology.task.install.deb:Package version is 20.2.0-712-g70f8415b-1jammy 2026-03-24T02:12:29.860 INFO:teuthology.task.install.deb:Pulling from https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default/ 2026-03-24T02:12:29.860 INFO:teuthology.task.install.deb:Package version is 20.2.0-712-g70f8415b-1jammy 2026-03-24T02:12:30.321 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:12:30.321 DEBUG:teuthology.orchestra.run.vm07:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2026-03-24T02:12:30.330 DEBUG:teuthology.orchestra.run.vm07:> sudo apt-get update 2026-03-24T02:12:30.401 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:12:30.401 DEBUG:teuthology.orchestra.run.vm06:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2026-03-24T02:12:30.410 DEBUG:teuthology.orchestra.run.vm06:> sudo apt-get update 2026-03-24T02:12:30.497 INFO:teuthology.orchestra.run.vm07.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2026-03-24T02:12:30.529 INFO:teuthology.orchestra.run.vm07.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2026-03-24T02:12:30.566 INFO:teuthology.orchestra.run.vm07.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2026-03-24T02:12:30.571 INFO:teuthology.orchestra.run.vm06.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2026-03-24T02:12:30.575 INFO:teuthology.orchestra.run.vm06.stdout:Hit:2 http://security.ubuntu.com/ubuntu jammy-security InRelease 2026-03-24T02:12:30.597 INFO:teuthology.orchestra.run.vm06.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2026-03-24T02:12:30.615 INFO:teuthology.orchestra.run.vm07.stdout:Hit:4 http://security.ubuntu.com/ubuntu jammy-security InRelease 2026-03-24T02:12:30.630 INFO:teuthology.orchestra.run.vm06.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2026-03-24T02:12:30.941 INFO:teuthology.orchestra.run.vm07.stdout:Ign:5 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy InRelease 2026-03-24T02:12:30.960 INFO:teuthology.orchestra.run.vm06.stdout:Ign:5 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy InRelease 2026-03-24T02:12:31.053 INFO:teuthology.orchestra.run.vm07.stdout:Get:6 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy Release [7680 B] 2026-03-24T02:12:31.072 INFO:teuthology.orchestra.run.vm06.stdout:Get:6 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy Release [7680 B] 2026-03-24T02:12:31.165 INFO:teuthology.orchestra.run.vm07.stdout:Ign:7 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy Release.gpg 2026-03-24T02:12:31.184 INFO:teuthology.orchestra.run.vm06.stdout:Ign:7 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy Release.gpg 2026-03-24T02:12:31.277 INFO:teuthology.orchestra.run.vm07.stdout:Get:8 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 Packages [18.8 kB] 2026-03-24T02:12:31.296 INFO:teuthology.orchestra.run.vm06.stdout:Get:8 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 Packages [18.8 kB] 2026-03-24T02:12:31.354 INFO:teuthology.orchestra.run.vm07.stdout:Fetched 26.5 kB in 1s (30.4 kB/s) 2026-03-24T02:12:31.494 INFO:teuthology.orchestra.run.vm06.stdout:Fetched 26.5 kB in 1s (28.7 kB/s) 2026-03-24T02:12:32.066 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:12:32.078 DEBUG:teuthology.orchestra.run.vm07:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=20.2.0-712-g70f8415b-1jammy cephadm=20.2.0-712-g70f8415b-1jammy ceph-mds=20.2.0-712-g70f8415b-1jammy ceph-mgr=20.2.0-712-g70f8415b-1jammy ceph-common=20.2.0-712-g70f8415b-1jammy ceph-fuse=20.2.0-712-g70f8415b-1jammy ceph-test=20.2.0-712-g70f8415b-1jammy ceph-volume=20.2.0-712-g70f8415b-1jammy radosgw=20.2.0-712-g70f8415b-1jammy python3-rados=20.2.0-712-g70f8415b-1jammy python3-rgw=20.2.0-712-g70f8415b-1jammy python3-cephfs=20.2.0-712-g70f8415b-1jammy python3-rbd=20.2.0-712-g70f8415b-1jammy libcephfs2=20.2.0-712-g70f8415b-1jammy libcephfs-dev=20.2.0-712-g70f8415b-1jammy librados2=20.2.0-712-g70f8415b-1jammy librbd1=20.2.0-712-g70f8415b-1jammy rbd-fuse=20.2.0-712-g70f8415b-1jammy rbd-mirror=20.2.0-712-g70f8415b-1jammy 2026-03-24T02:12:32.112 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:12:32.201 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:12:32.214 DEBUG:teuthology.orchestra.run.vm06:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=20.2.0-712-g70f8415b-1jammy cephadm=20.2.0-712-g70f8415b-1jammy ceph-mds=20.2.0-712-g70f8415b-1jammy ceph-mgr=20.2.0-712-g70f8415b-1jammy ceph-common=20.2.0-712-g70f8415b-1jammy ceph-fuse=20.2.0-712-g70f8415b-1jammy ceph-test=20.2.0-712-g70f8415b-1jammy ceph-volume=20.2.0-712-g70f8415b-1jammy radosgw=20.2.0-712-g70f8415b-1jammy python3-rados=20.2.0-712-g70f8415b-1jammy python3-rgw=20.2.0-712-g70f8415b-1jammy python3-cephfs=20.2.0-712-g70f8415b-1jammy python3-rbd=20.2.0-712-g70f8415b-1jammy libcephfs2=20.2.0-712-g70f8415b-1jammy libcephfs-dev=20.2.0-712-g70f8415b-1jammy librados2=20.2.0-712-g70f8415b-1jammy librbd1=20.2.0-712-g70f8415b-1jammy rbd-fuse=20.2.0-712-g70f8415b-1jammy rbd-mirror=20.2.0-712-g70f8415b-1jammy 2026-03-24T02:12:32.258 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:12:32.283 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:12:32.286 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:12:32.404 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:12:32.404 INFO:teuthology.orchestra.run.vm07.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libpmemobj1 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: libsgutils2-2 sg3-utils sg3-utils-udev 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout:The following additional packages will be installed: 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: libcephfs-daemon libcephfs-proxy2 libdouble-conversion3 libfuse2 libjq1 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: liblttng-ust1 libnbd0 liboath0 libonig5 libpcre2-16-0 libqt5core5a 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: libqt5dbus5 libqt5network5 libradosstriper1 librdkafka1 librgw2 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli python-asyncssh-doc 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: python3-asyncssh python3-cachetools python3-ceph-argparse 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: python3-ceph-common python3-cheroot python3-cherrypy3 python3-google-auth 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: python3-kubernetes python3-natsort python3-pluggy python3-portend 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: python3-prettytable python3-psutil python3-py python3-pygments 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: python3-pytest python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: python3-tempora python3-threadpoolctl python3-toml python3-wcwidth 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: python3-webob python3-websocket python3-zc.lockfile qttranslations5-l10n 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: smartmontools socat xmlstarlet 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout:Suggested packages: 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: python3-influxdb liblua5.3-dev luarocks python-natsort-doc python-psutil-doc 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: subversion python-pygments-doc ttf-bitstream-vera python3-paste python3-dap 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: python-sklearn-doc ipython3 python-webob-doc gsmartcontrol smart-notifier 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: mailx | mailutils 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout:Recommended packages: 2026-03-24T02:12:32.405 INFO:teuthology.orchestra.run.vm07.stdout: btrfs-tools 2026-03-24T02:12:32.440 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:12:32.441 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:12:32.447 INFO:teuthology.orchestra.run.vm07.stdout:The following NEW packages will be installed: 2026-03-24T02:12:32.447 INFO:teuthology.orchestra.run.vm07.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2026-03-24T02:12:32.447 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2026-03-24T02:12:32.447 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2026-03-24T02:12:32.447 INFO:teuthology.orchestra.run.vm07.stdout: libcephfs-daemon libcephfs-dev libcephfs-proxy2 libcephfs2 2026-03-24T02:12:32.447 INFO:teuthology.orchestra.run.vm07.stdout: libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 libnbd0 liboath0 2026-03-24T02:12:32.447 INFO:teuthology.orchestra.run.vm07.stdout: libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2026-03-24T02:12:32.448 INFO:teuthology.orchestra.run.vm07.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 2026-03-24T02:12:32.448 INFO:teuthology.orchestra.run.vm07.stdout: nvme-cli python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:12:32.448 INFO:teuthology.orchestra.run.vm07.stdout: python3-ceph-argparse python3-ceph-common python3-cephfs python3-cheroot 2026-03-24T02:12:32.448 INFO:teuthology.orchestra.run.vm07.stdout: python3-cherrypy3 python3-google-auth python3-iniconfig 2026-03-24T02:12:32.448 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2026-03-24T02:12:32.448 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.text python3-joblib python3-kubernetes python3-natsort 2026-03-24T02:12:32.448 INFO:teuthology.orchestra.run.vm07.stdout: python3-pluggy python3-portend python3-prettytable python3-psutil python3-py 2026-03-24T02:12:32.448 INFO:teuthology.orchestra.run.vm07.stdout: python3-pygments python3-pytest python3-rados python3-rbd python3-repoze.lru 2026-03-24T02:12:32.448 INFO:teuthology.orchestra.run.vm07.stdout: python3-requests-oauthlib python3-rgw python3-routes python3-rsa 2026-03-24T02:12:32.448 INFO:teuthology.orchestra.run.vm07.stdout: python3-simplejson python3-sklearn python3-sklearn-lib python3-tempora 2026-03-24T02:12:32.448 INFO:teuthology.orchestra.run.vm07.stdout: python3-threadpoolctl python3-toml python3-wcwidth python3-webob 2026-03-24T02:12:32.448 INFO:teuthology.orchestra.run.vm07.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse 2026-03-24T02:12:32.448 INFO:teuthology.orchestra.run.vm07.stdout: rbd-mirror smartmontools socat xmlstarlet 2026-03-24T02:12:32.448 INFO:teuthology.orchestra.run.vm07.stdout:The following packages will be upgraded: 2026-03-24T02:12:32.449 INFO:teuthology.orchestra.run.vm07.stdout: librados2 librbd1 2026-03-24T02:12:32.601 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:12:32.601 INFO:teuthology.orchestra.run.vm06.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libpmemobj1 2026-03-24T02:12:32.601 INFO:teuthology.orchestra.run.vm06.stdout: libsgutils2-2 sg3-utils sg3-utils-udev 2026-03-24T02:12:32.601 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:12:32.601 INFO:teuthology.orchestra.run.vm06.stdout:The following additional packages will be installed: 2026-03-24T02:12:32.601 INFO:teuthology.orchestra.run.vm06.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2026-03-24T02:12:32.601 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq 2026-03-24T02:12:32.601 INFO:teuthology.orchestra.run.vm06.stdout: libcephfs-daemon libcephfs-proxy2 libdouble-conversion3 libfuse2 libjq1 2026-03-24T02:12:32.601 INFO:teuthology.orchestra.run.vm06.stdout: liblttng-ust1 libnbd0 liboath0 libonig5 libpcre2-16-0 libqt5core5a 2026-03-24T02:12:32.601 INFO:teuthology.orchestra.run.vm06.stdout: libqt5dbus5 libqt5network5 libradosstriper1 librdkafka1 librgw2 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout: libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli python-asyncssh-doc 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout: python3-asyncssh python3-cachetools python3-ceph-argparse 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout: python3-ceph-common python3-cheroot python3-cherrypy3 python3-google-auth 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout: python3-kubernetes python3-natsort python3-pluggy python3-portend 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout: python3-prettytable python3-psutil python3-py python3-pygments 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout: python3-pytest python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout: python3-tempora python3-threadpoolctl python3-toml python3-wcwidth 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout: python3-webob python3-websocket python3-zc.lockfile qttranslations5-l10n 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout: smartmontools socat xmlstarlet 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout:Suggested packages: 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout: python3-influxdb liblua5.3-dev luarocks python-natsort-doc python-psutil-doc 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout: subversion python-pygments-doc ttf-bitstream-vera python3-paste python3-dap 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout: python-sklearn-doc ipython3 python-webob-doc gsmartcontrol smart-notifier 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout: mailx | mailutils 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout:Recommended packages: 2026-03-24T02:12:32.602 INFO:teuthology.orchestra.run.vm06.stdout: btrfs-tools 2026-03-24T02:12:32.644 INFO:teuthology.orchestra.run.vm06.stdout:The following NEW packages will be installed: 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: libcephfs-daemon libcephfs-dev libcephfs-proxy2 libcephfs2 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 libnbd0 liboath0 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: nvme-cli python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: python3-ceph-argparse python3-ceph-common python3-cephfs python3-cheroot 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: python3-cherrypy3 python3-google-auth python3-iniconfig 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.text python3-joblib python3-kubernetes python3-natsort 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: python3-pluggy python3-portend python3-prettytable python3-psutil python3-py 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: python3-pygments python3-pytest python3-rados python3-rbd python3-repoze.lru 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: python3-requests-oauthlib python3-rgw python3-routes python3-rsa 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: python3-simplejson python3-sklearn python3-sklearn-lib python3-tempora 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: python3-threadpoolctl python3-toml python3-wcwidth python3-webob 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout: rbd-mirror smartmontools socat xmlstarlet 2026-03-24T02:12:32.645 INFO:teuthology.orchestra.run.vm06.stdout:The following packages will be upgraded: 2026-03-24T02:12:32.646 INFO:teuthology.orchestra.run.vm06.stdout: librados2 librbd1 2026-03-24T02:12:32.669 INFO:teuthology.orchestra.run.vm07.stdout:2 upgraded, 86 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:12:32.669 INFO:teuthology.orchestra.run.vm07.stdout:Need to get 283 MB of archives. 2026-03-24T02:12:32.669 INFO:teuthology.orchestra.run.vm07.stdout:After this operation, 1103 MB of additional disk space will be used. 2026-03-24T02:12:32.669 INFO:teuthology.orchestra.run.vm07.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2026-03-24T02:12:32.723 INFO:teuthology.orchestra.run.vm06.stdout:2 upgraded, 86 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:12:32.723 INFO:teuthology.orchestra.run.vm06.stdout:Need to get 283 MB of archives. 2026-03-24T02:12:32.723 INFO:teuthology.orchestra.run.vm06.stdout:After this operation, 1103 MB of additional disk space will be used. 2026-03-24T02:12:32.723 INFO:teuthology.orchestra.run.vm06.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2026-03-24T02:12:32.882 INFO:teuthology.orchestra.run.vm06.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2026-03-24T02:12:32.886 INFO:teuthology.orchestra.run.vm06.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2026-03-24T02:12:32.918 INFO:teuthology.orchestra.run.vm06.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2006 kB] 2026-03-24T02:12:33.008 INFO:teuthology.orchestra.run.vm06.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2026-03-24T02:12:33.012 INFO:teuthology.orchestra.run.vm06.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2026-03-24T02:12:33.024 INFO:teuthology.orchestra.run.vm06.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2026-03-24T02:12:33.028 INFO:teuthology.orchestra.run.vm06.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libnbd0 amd64 1.10.5-1 [71.3 kB] 2026-03-24T02:12:33.028 INFO:teuthology.orchestra.run.vm06.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2026-03-24T02:12:33.029 INFO:teuthology.orchestra.run.vm06.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2026-03-24T02:12:33.029 INFO:teuthology.orchestra.run.vm06.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2026-03-24T02:12:33.037 INFO:teuthology.orchestra.run.vm06.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 liboath0 amd64 2.6.7-3ubuntu0.1 [41.3 kB] 2026-03-24T02:12:33.037 INFO:teuthology.orchestra.run.vm06.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9030 B] 2026-03-24T02:12:33.040 INFO:teuthology.orchestra.run.vm06.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2026-03-24T02:12:33.050 INFO:teuthology.orchestra.run.vm07.stdout:Get:2 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 20.2.0-712-g70f8415b-1jammy [2867 kB] 2026-03-24T02:12:33.069 INFO:teuthology.orchestra.run.vm06.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6452 B] 2026-03-24T02:12:33.069 INFO:teuthology.orchestra.run.vm06.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8716 B] 2026-03-24T02:12:33.070 INFO:teuthology.orchestra.run.vm06.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2026-03-24T02:12:33.070 INFO:teuthology.orchestra.run.vm06.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2026-03-24T02:12:33.070 INFO:teuthology.orchestra.run.vm06.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7240 B] 2026-03-24T02:12:33.070 INFO:teuthology.orchestra.run.vm06.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8980 B] 2026-03-24T02:12:33.070 INFO:teuthology.orchestra.run.vm06.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2026-03-24T02:12:33.071 INFO:teuthology.orchestra.run.vm06.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2026-03-24T02:12:33.072 INFO:teuthology.orchestra.run.vm06.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfuse2 amd64 2.9.9-5ubuntu3 [90.3 kB] 2026-03-24T02:12:33.104 INFO:teuthology.orchestra.run.vm06.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python3-asyncssh all 2.5.0-1ubuntu0.1 [189 kB] 2026-03-24T02:12:33.105 INFO:teuthology.orchestra.run.vm06.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2026-03-24T02:12:33.105 INFO:teuthology.orchestra.run.vm06.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2026-03-24T02:12:33.106 INFO:teuthology.orchestra.run.vm06.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2058 kB] 2026-03-24T02:12:33.170 INFO:teuthology.orchestra.run.vm06.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2026-03-24T02:12:33.171 INFO:teuthology.orchestra.run.vm06.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2026-03-24T02:12:33.171 INFO:teuthology.orchestra.run.vm06.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1829 kB] 2026-03-24T02:12:33.178 INFO:teuthology.orchestra.run.vm06.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9722 B] 2026-03-24T02:12:33.178 INFO:teuthology.orchestra.run.vm06.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2026-03-24T02:12:33.178 INFO:teuthology.orchestra.run.vm06.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2026-03-24T02:12:33.178 INFO:teuthology.orchestra.run.vm06.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2026-03-24T02:12:33.179 INFO:teuthology.orchestra.run.vm06.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2026-03-24T02:12:33.181 INFO:teuthology.orchestra.run.vm07.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2026-03-24T02:12:33.197 INFO:teuthology.orchestra.run.vm07.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2026-03-24T02:12:33.202 INFO:teuthology.orchestra.run.vm06.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2026-03-24T02:12:33.205 INFO:teuthology.orchestra.run.vm06.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2026-03-24T02:12:33.237 INFO:teuthology.orchestra.run.vm06.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libjq1 amd64 1.6-2.1ubuntu3.1 [133 kB] 2026-03-24T02:12:33.238 INFO:teuthology.orchestra.run.vm06.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 jq amd64 1.6-2.1ubuntu3.1 [52.5 kB] 2026-03-24T02:12:33.238 INFO:teuthology.orchestra.run.vm06.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2026-03-24T02:12:33.241 INFO:teuthology.orchestra.run.vm06.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2026-03-24T02:12:33.243 INFO:teuthology.orchestra.run.vm06.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2026-03-24T02:12:33.247 INFO:teuthology.orchestra.run.vm06.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python-asyncssh-doc all 2.5.0-1ubuntu0.1 [309 kB] 2026-03-24T02:12:33.262 INFO:teuthology.orchestra.run.vm06.stdout:Get:44 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 20.2.0-712-g70f8415b-1jammy [2867 kB] 2026-03-24T02:12:33.294 INFO:teuthology.orchestra.run.vm06.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6024 B] 2026-03-24T02:12:33.294 INFO:teuthology.orchestra.run.vm06.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2026-03-24T02:12:33.294 INFO:teuthology.orchestra.run.vm06.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2026-03-24T02:12:33.295 INFO:teuthology.orchestra.run.vm06.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2026-03-24T02:12:33.295 INFO:teuthology.orchestra.run.vm07.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2006 kB] 2026-03-24T02:12:33.299 INFO:teuthology.orchestra.run.vm06.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-pygments all 2.11.2+dfsg-2ubuntu0.1 [750 kB] 2026-03-24T02:12:33.329 INFO:teuthology.orchestra.run.vm06.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2026-03-24T02:12:33.329 INFO:teuthology.orchestra.run.vm06.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2026-03-24T02:12:33.330 INFO:teuthology.orchestra.run.vm06.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2026-03-24T02:12:33.331 INFO:teuthology.orchestra.run.vm06.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2026-03-24T02:12:33.331 INFO:teuthology.orchestra.run.vm06.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1983 kB] 2026-03-24T02:12:33.420 INFO:teuthology.orchestra.run.vm06.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 smartmontools amd64 7.2-1ubuntu0.1 [583 kB] 2026-03-24T02:12:33.648 INFO:teuthology.orchestra.run.vm07.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2026-03-24T02:12:33.664 INFO:teuthology.orchestra.run.vm07.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2026-03-24T02:12:33.710 INFO:teuthology.orchestra.run.vm07.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2026-03-24T02:12:33.723 INFO:teuthology.orchestra.run.vm07.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libnbd0 amd64 1.10.5-1 [71.3 kB] 2026-03-24T02:12:33.725 INFO:teuthology.orchestra.run.vm07.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2026-03-24T02:12:33.726 INFO:teuthology.orchestra.run.vm07.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2026-03-24T02:12:33.728 INFO:teuthology.orchestra.run.vm07.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2026-03-24T02:12:33.754 INFO:teuthology.orchestra.run.vm07.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 liboath0 amd64 2.6.7-3ubuntu0.1 [41.3 kB] 2026-03-24T02:12:33.755 INFO:teuthology.orchestra.run.vm07.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9030 B] 2026-03-24T02:12:33.756 INFO:teuthology.orchestra.run.vm07.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2026-03-24T02:12:33.852 INFO:teuthology.orchestra.run.vm07.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6452 B] 2026-03-24T02:12:33.852 INFO:teuthology.orchestra.run.vm07.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8716 B] 2026-03-24T02:12:33.853 INFO:teuthology.orchestra.run.vm07.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2026-03-24T02:12:33.853 INFO:teuthology.orchestra.run.vm07.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2026-03-24T02:12:33.853 INFO:teuthology.orchestra.run.vm07.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7240 B] 2026-03-24T02:12:33.853 INFO:teuthology.orchestra.run.vm07.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8980 B] 2026-03-24T02:12:33.868 INFO:teuthology.orchestra.run.vm07.stdout:Get:22 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 20.2.0-712-g70f8415b-1jammy [3583 kB] 2026-03-24T02:12:33.953 INFO:teuthology.orchestra.run.vm07.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2026-03-24T02:12:33.956 INFO:teuthology.orchestra.run.vm07.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2026-03-24T02:12:33.956 INFO:teuthology.orchestra.run.vm07.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfuse2 amd64 2.9.9-5ubuntu3 [90.3 kB] 2026-03-24T02:12:33.958 INFO:teuthology.orchestra.run.vm07.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python3-asyncssh all 2.5.0-1ubuntu0.1 [189 kB] 2026-03-24T02:12:33.995 INFO:teuthology.orchestra.run.vm07.stdout:Get:27 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 20.2.0-712-g70f8415b-1jammy [829 kB] 2026-03-24T02:12:34.009 INFO:teuthology.orchestra.run.vm07.stdout:Get:28 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 20.2.0-712-g70f8415b-1jammy [364 kB] 2026-03-24T02:12:34.014 INFO:teuthology.orchestra.run.vm07.stdout:Get:29 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 20.2.0-712-g70f8415b-1jammy [32.8 kB] 2026-03-24T02:12:34.015 INFO:teuthology.orchestra.run.vm07.stdout:Get:30 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 20.2.0-712-g70f8415b-1jammy [184 kB] 2026-03-24T02:12:34.018 INFO:teuthology.orchestra.run.vm07.stdout:Get:31 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 20.2.0-712-g70f8415b-1jammy [83.8 kB] 2026-03-24T02:12:34.019 INFO:teuthology.orchestra.run.vm07.stdout:Get:32 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 20.2.0-712-g70f8415b-1jammy [341 kB] 2026-03-24T02:12:34.025 INFO:teuthology.orchestra.run.vm07.stdout:Get:33 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 20.2.0-712-g70f8415b-1jammy [8697 kB] 2026-03-24T02:12:34.055 INFO:teuthology.orchestra.run.vm07.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2026-03-24T02:12:34.055 INFO:teuthology.orchestra.run.vm07.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2026-03-24T02:12:34.056 INFO:teuthology.orchestra.run.vm07.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2058 kB] 2026-03-24T02:12:34.094 INFO:teuthology.orchestra.run.vm07.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2026-03-24T02:12:34.098 INFO:teuthology.orchestra.run.vm07.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2026-03-24T02:12:34.098 INFO:teuthology.orchestra.run.vm07.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1829 kB] 2026-03-24T02:12:34.103 INFO:teuthology.orchestra.run.vm06.stdout:Get:56 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 20.2.0-712-g70f8415b-1jammy [3583 kB] 2026-03-24T02:12:34.178 INFO:teuthology.orchestra.run.vm07.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9722 B] 2026-03-24T02:12:34.178 INFO:teuthology.orchestra.run.vm07.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2026-03-24T02:12:34.178 INFO:teuthology.orchestra.run.vm07.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2026-03-24T02:12:34.178 INFO:teuthology.orchestra.run.vm07.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2026-03-24T02:12:34.230 INFO:teuthology.orchestra.run.vm06.stdout:Get:57 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 20.2.0-712-g70f8415b-1jammy [829 kB] 2026-03-24T02:12:34.240 INFO:teuthology.orchestra.run.vm06.stdout:Get:58 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 20.2.0-712-g70f8415b-1jammy [364 kB] 2026-03-24T02:12:34.246 INFO:teuthology.orchestra.run.vm06.stdout:Get:59 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 20.2.0-712-g70f8415b-1jammy [32.8 kB] 2026-03-24T02:12:34.246 INFO:teuthology.orchestra.run.vm06.stdout:Get:60 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 20.2.0-712-g70f8415b-1jammy [184 kB] 2026-03-24T02:12:34.249 INFO:teuthology.orchestra.run.vm06.stdout:Get:61 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 20.2.0-712-g70f8415b-1jammy [83.8 kB] 2026-03-24T02:12:34.250 INFO:teuthology.orchestra.run.vm06.stdout:Get:62 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 20.2.0-712-g70f8415b-1jammy [341 kB] 2026-03-24T02:12:34.256 INFO:teuthology.orchestra.run.vm06.stdout:Get:63 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 20.2.0-712-g70f8415b-1jammy [8697 kB] 2026-03-24T02:12:34.257 INFO:teuthology.orchestra.run.vm07.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2026-03-24T02:12:34.258 INFO:teuthology.orchestra.run.vm07.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2026-03-24T02:12:34.264 INFO:teuthology.orchestra.run.vm07.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2026-03-24T02:12:34.266 INFO:teuthology.orchestra.run.vm07.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libjq1 amd64 1.6-2.1ubuntu3.1 [133 kB] 2026-03-24T02:12:34.268 INFO:teuthology.orchestra.run.vm07.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 jq amd64 1.6-2.1ubuntu3.1 [52.5 kB] 2026-03-24T02:12:34.358 INFO:teuthology.orchestra.run.vm07.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2026-03-24T02:12:34.363 INFO:teuthology.orchestra.run.vm07.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2026-03-24T02:12:34.366 INFO:teuthology.orchestra.run.vm07.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2026-03-24T02:12:34.373 INFO:teuthology.orchestra.run.vm07.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python-asyncssh-doc all 2.5.0-1ubuntu0.1 [309 kB] 2026-03-24T02:12:34.378 INFO:teuthology.orchestra.run.vm07.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6024 B] 2026-03-24T02:12:34.416 INFO:teuthology.orchestra.run.vm07.stdout:Get:54 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 20.2.0-712-g70f8415b-1jammy [112 kB] 2026-03-24T02:12:34.417 INFO:teuthology.orchestra.run.vm07.stdout:Get:55 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 20.2.0-712-g70f8415b-1jammy [261 kB] 2026-03-24T02:12:34.421 INFO:teuthology.orchestra.run.vm07.stdout:Get:56 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 20.2.0-712-g70f8415b-1jammy [29.3 MB] 2026-03-24T02:12:34.458 INFO:teuthology.orchestra.run.vm07.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2026-03-24T02:12:34.458 INFO:teuthology.orchestra.run.vm07.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2026-03-24T02:12:34.460 INFO:teuthology.orchestra.run.vm07.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2026-03-24T02:12:34.461 INFO:teuthology.orchestra.run.vm07.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-pygments all 2.11.2+dfsg-2ubuntu0.1 [750 kB] 2026-03-24T02:12:34.474 INFO:teuthology.orchestra.run.vm07.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2026-03-24T02:12:34.559 INFO:teuthology.orchestra.run.vm07.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2026-03-24T02:12:34.561 INFO:teuthology.orchestra.run.vm07.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2026-03-24T02:12:34.561 INFO:teuthology.orchestra.run.vm07.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2026-03-24T02:12:34.563 INFO:teuthology.orchestra.run.vm07.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1983 kB] 2026-03-24T02:12:34.585 INFO:teuthology.orchestra.run.vm07.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 smartmontools amd64 7.2-1ubuntu0.1 [583 kB] 2026-03-24T02:12:34.615 INFO:teuthology.orchestra.run.vm06.stdout:Get:64 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 20.2.0-712-g70f8415b-1jammy [112 kB] 2026-03-24T02:12:34.618 INFO:teuthology.orchestra.run.vm06.stdout:Get:65 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 20.2.0-712-g70f8415b-1jammy [261 kB] 2026-03-24T02:12:34.621 INFO:teuthology.orchestra.run.vm06.stdout:Get:66 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 20.2.0-712-g70f8415b-1jammy [29.3 MB] 2026-03-24T02:12:35.591 INFO:teuthology.orchestra.run.vm07.stdout:Get:67 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 20.2.0-712-g70f8415b-1jammy [5415 kB] 2026-03-24T02:12:35.762 INFO:teuthology.orchestra.run.vm07.stdout:Get:68 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 20.2.0-712-g70f8415b-1jammy [246 kB] 2026-03-24T02:12:35.766 INFO:teuthology.orchestra.run.vm07.stdout:Get:69 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 20.2.0-712-g70f8415b-1jammy [124 kB] 2026-03-24T02:12:35.772 INFO:teuthology.orchestra.run.vm07.stdout:Get:70 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 20.2.0-712-g70f8415b-1jammy [906 kB] 2026-03-24T02:12:35.794 INFO:teuthology.orchestra.run.vm06.stdout:Get:67 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 20.2.0-712-g70f8415b-1jammy [5415 kB] 2026-03-24T02:12:35.860 INFO:teuthology.orchestra.run.vm07.stdout:Get:71 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 20.2.0-712-g70f8415b-1jammy [6399 kB] 2026-03-24T02:12:35.976 INFO:teuthology.orchestra.run.vm06.stdout:Get:68 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 20.2.0-712-g70f8415b-1jammy [246 kB] 2026-03-24T02:12:36.014 INFO:teuthology.orchestra.run.vm06.stdout:Get:69 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 20.2.0-712-g70f8415b-1jammy [124 kB] 2026-03-24T02:12:36.015 INFO:teuthology.orchestra.run.vm06.stdout:Get:70 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 20.2.0-712-g70f8415b-1jammy [906 kB] 2026-03-24T02:12:36.055 INFO:teuthology.orchestra.run.vm06.stdout:Get:71 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 20.2.0-712-g70f8415b-1jammy [6399 kB] 2026-03-24T02:12:36.100 INFO:teuthology.orchestra.run.vm07.stdout:Get:72 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 20.2.0-712-g70f8415b-1jammy [21.7 MB] 2026-03-24T02:12:36.342 INFO:teuthology.orchestra.run.vm06.stdout:Get:72 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 20.2.0-712-g70f8415b-1jammy [21.7 MB] 2026-03-24T02:12:36.909 INFO:teuthology.orchestra.run.vm07.stdout:Get:73 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 20.2.0-712-g70f8415b-1jammy [14.1 kB] 2026-03-24T02:12:36.924 INFO:teuthology.orchestra.run.vm07.stdout:Get:74 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 20.2.0-712-g70f8415b-1jammy [955 kB] 2026-03-24T02:12:36.965 INFO:teuthology.orchestra.run.vm07.stdout:Get:75 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 20.2.0-712-g70f8415b-1jammy [2341 kB] 2026-03-24T02:12:37.055 INFO:teuthology.orchestra.run.vm07.stdout:Get:76 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 20.2.0-712-g70f8415b-1jammy [1049 kB] 2026-03-24T02:12:37.089 INFO:teuthology.orchestra.run.vm07.stdout:Get:77 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 20.2.0-712-g70f8415b-1jammy [179 kB] 2026-03-24T02:12:37.092 INFO:teuthology.orchestra.run.vm07.stdout:Get:78 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 20.2.0-712-g70f8415b-1jammy [45.5 MB] 2026-03-24T02:12:37.192 INFO:teuthology.orchestra.run.vm06.stdout:Get:73 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 20.2.0-712-g70f8415b-1jammy [14.1 kB] 2026-03-24T02:12:37.192 INFO:teuthology.orchestra.run.vm06.stdout:Get:74 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 20.2.0-712-g70f8415b-1jammy [955 kB] 2026-03-24T02:12:37.231 INFO:teuthology.orchestra.run.vm06.stdout:Get:75 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 20.2.0-712-g70f8415b-1jammy [2341 kB] 2026-03-24T02:12:37.315 INFO:teuthology.orchestra.run.vm06.stdout:Get:76 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 20.2.0-712-g70f8415b-1jammy [1049 kB] 2026-03-24T02:12:37.378 INFO:teuthology.orchestra.run.vm06.stdout:Get:77 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 20.2.0-712-g70f8415b-1jammy [179 kB] 2026-03-24T02:12:37.385 INFO:teuthology.orchestra.run.vm06.stdout:Get:78 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 20.2.0-712-g70f8415b-1jammy [45.5 MB] 2026-03-24T02:12:38.874 INFO:teuthology.orchestra.run.vm07.stdout:Get:79 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 20.2.0-712-g70f8415b-1jammy [8625 kB] 2026-03-24T02:12:39.167 INFO:teuthology.orchestra.run.vm07.stdout:Get:80 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 20.2.0-712-g70f8415b-1jammy [14.2 kB] 2026-03-24T02:12:39.167 INFO:teuthology.orchestra.run.vm07.stdout:Get:81 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 20.2.0-712-g70f8415b-1jammy [99.5 MB] 2026-03-24T02:12:39.173 INFO:teuthology.orchestra.run.vm06.stdout:Get:79 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 20.2.0-712-g70f8415b-1jammy [8625 kB] 2026-03-24T02:12:39.495 INFO:teuthology.orchestra.run.vm06.stdout:Get:80 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 20.2.0-712-g70f8415b-1jammy [14.2 kB] 2026-03-24T02:12:39.496 INFO:teuthology.orchestra.run.vm06.stdout:Get:81 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 20.2.0-712-g70f8415b-1jammy [99.5 MB] 2026-03-24T02:12:43.337 INFO:teuthology.orchestra.run.vm07.stdout:Get:82 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 20.2.0-712-g70f8415b-1jammy [135 kB] 2026-03-24T02:12:43.337 INFO:teuthology.orchestra.run.vm07.stdout:Get:83 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-daemon amd64 20.2.0-712-g70f8415b-1jammy [43.3 kB] 2026-03-24T02:12:43.337 INFO:teuthology.orchestra.run.vm07.stdout:Get:84 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-proxy2 amd64 20.2.0-712-g70f8415b-1jammy [30.7 kB] 2026-03-24T02:12:43.337 INFO:teuthology.orchestra.run.vm07.stdout:Get:85 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 20.2.0-712-g70f8415b-1jammy [41.5 kB] 2026-03-24T02:12:43.337 INFO:teuthology.orchestra.run.vm07.stdout:Get:86 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 20.2.0-712-g70f8415b-1jammy [25.1 MB] 2026-03-24T02:12:43.909 INFO:teuthology.orchestra.run.vm06.stdout:Get:82 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 20.2.0-712-g70f8415b-1jammy [135 kB] 2026-03-24T02:12:43.909 INFO:teuthology.orchestra.run.vm06.stdout:Get:83 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-daemon amd64 20.2.0-712-g70f8415b-1jammy [43.3 kB] 2026-03-24T02:12:43.912 INFO:teuthology.orchestra.run.vm06.stdout:Get:84 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-proxy2 amd64 20.2.0-712-g70f8415b-1jammy [30.7 kB] 2026-03-24T02:12:43.912 INFO:teuthology.orchestra.run.vm06.stdout:Get:85 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 20.2.0-712-g70f8415b-1jammy [41.5 kB] 2026-03-24T02:12:43.912 INFO:teuthology.orchestra.run.vm06.stdout:Get:86 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 20.2.0-712-g70f8415b-1jammy [25.1 MB] 2026-03-24T02:12:44.280 INFO:teuthology.orchestra.run.vm07.stdout:Get:87 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 20.2.0-712-g70f8415b-1jammy [97.9 kB] 2026-03-24T02:12:44.281 INFO:teuthology.orchestra.run.vm07.stdout:Get:88 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 20.2.0-712-g70f8415b-1jammy [2883 kB] 2026-03-24T02:12:44.687 INFO:teuthology.orchestra.run.vm07.stdout:Fetched 283 MB in 12s (23.8 MB/s) 2026-03-24T02:12:44.862 INFO:teuthology.orchestra.run.vm06.stdout:Get:87 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 20.2.0-712-g70f8415b-1jammy [97.9 kB] 2026-03-24T02:12:44.867 INFO:teuthology.orchestra.run.vm06.stdout:Get:88 https://1.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 20.2.0-712-g70f8415b-1jammy [2883 kB] 2026-03-24T02:12:44.918 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2026-03-24T02:12:44.953 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 119262 files and directories currently installed.) 2026-03-24T02:12:44.955 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../00-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2026-03-24T02:12:44.957 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-03-24T02:12:44.978 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2026-03-24T02:12:44.984 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../01-libdouble-conversion3_3.1.7-4_amd64.deb ... 2026-03-24T02:12:44.985 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2026-03-24T02:12:45.002 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2026-03-24T02:12:45.008 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../02-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2026-03-24T02:12:45.009 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-03-24T02:12:45.029 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package libqt5core5a:amd64. 2026-03-24T02:12:45.036 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../03-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-03-24T02:12:45.040 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:12:45.087 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2026-03-24T02:12:45.094 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../04-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-03-24T02:12:45.096 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:12:45.126 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package libqt5network5:amd64. 2026-03-24T02:12:45.132 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../05-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-03-24T02:12:45.133 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:12:45.162 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2026-03-24T02:12:45.170 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../06-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2026-03-24T02:12:45.170 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-03-24T02:12:45.200 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../07-librbd1_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:45.203 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking librbd1 (20.2.0-712-g70f8415b-1jammy) over (17.2.9-0ubuntu0.22.04.2) ... 2026-03-24T02:12:45.261 INFO:teuthology.orchestra.run.vm06.stdout:Fetched 283 MB in 12s (23.0 MB/s) 2026-03-24T02:12:45.480 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2026-03-24T02:12:45.485 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../08-librados2_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:45.499 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking librados2 (20.2.0-712-g70f8415b-1jammy) over (17.2.9-0ubuntu0.22.04.2) ... 2026-03-24T02:12:45.513 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 119262 files and directories currently installed.) 2026-03-24T02:12:45.516 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../00-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2026-03-24T02:12:45.518 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-03-24T02:12:45.554 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2026-03-24T02:12:45.561 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../01-libdouble-conversion3_3.1.7-4_amd64.deb ... 2026-03-24T02:12:45.574 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2026-03-24T02:12:45.582 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package libnbd0. 2026-03-24T02:12:45.588 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../09-libnbd0_1.10.5-1_amd64.deb ... 2026-03-24T02:12:45.589 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking libnbd0 (1.10.5-1) ... 2026-03-24T02:12:45.591 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2026-03-24T02:12:45.599 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../02-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2026-03-24T02:12:45.599 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-03-24T02:12:45.606 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package libcephfs2. 2026-03-24T02:12:45.612 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../10-libcephfs2_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:45.613 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking libcephfs2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:45.621 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package libqt5core5a:amd64. 2026-03-24T02:12:45.628 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../03-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-03-24T02:12:45.633 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:12:45.640 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-rados. 2026-03-24T02:12:45.646 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../11-python3-rados_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:45.647 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-rados (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:45.677 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-ceph-argparse. 2026-03-24T02:12:45.681 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2026-03-24T02:12:45.684 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../12-python3-ceph-argparse_20.2.0-712-g70f8415b-1jammy_all.deb ... 2026-03-24T02:12:45.685 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-ceph-argparse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:45.687 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../04-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-03-24T02:12:45.688 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:12:45.701 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-cephfs. 2026-03-24T02:12:45.707 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../13-python3-cephfs_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:45.707 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package libqt5network5:amd64. 2026-03-24T02:12:45.708 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-cephfs (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:45.715 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../05-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-03-24T02:12:45.729 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:12:45.743 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-ceph-common. 2026-03-24T02:12:45.749 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../14-python3-ceph-common_20.2.0-712-g70f8415b-1jammy_all.deb ... 2026-03-24T02:12:45.750 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-ceph-common (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:45.755 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2026-03-24T02:12:45.761 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../06-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2026-03-24T02:12:45.762 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-03-24T02:12:45.770 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-wcwidth. 2026-03-24T02:12:45.776 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../15-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2026-03-24T02:12:45.777 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2026-03-24T02:12:45.789 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../07-librbd1_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:45.791 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking librbd1 (20.2.0-712-g70f8415b-1jammy) over (17.2.9-0ubuntu0.22.04.2) ... 2026-03-24T02:12:45.793 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-prettytable. 2026-03-24T02:12:45.799 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../16-python3-prettytable_2.5.0-2_all.deb ... 2026-03-24T02:12:45.800 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2026-03-24T02:12:45.814 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-rbd. 2026-03-24T02:12:45.820 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../17-python3-rbd_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:45.821 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-rbd (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:45.857 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package librdkafka1:amd64. 2026-03-24T02:12:45.863 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../18-librdkafka1_1.8.0-1build1_amd64.deb ... 2026-03-24T02:12:45.863 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2026-03-24T02:12:45.871 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../08-librados2_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:45.876 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking librados2 (20.2.0-712-g70f8415b-1jammy) over (17.2.9-0ubuntu0.22.04.2) ... 2026-03-24T02:12:45.894 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package librgw2. 2026-03-24T02:12:45.899 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../19-librgw2_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:45.900 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking librgw2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:45.946 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package libnbd0. 2026-03-24T02:12:45.952 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../09-libnbd0_1.10.5-1_amd64.deb ... 2026-03-24T02:12:45.954 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking libnbd0 (1.10.5-1) ... 2026-03-24T02:12:45.969 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package libcephfs2. 2026-03-24T02:12:45.975 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../10-libcephfs2_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:45.976 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking libcephfs2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:46.003 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-rados. 2026-03-24T02:12:46.009 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../11-python3-rados_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:46.057 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-rados (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:46.069 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-rgw. 2026-03-24T02:12:46.076 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../20-python3-rgw_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:46.076 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-rgw (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:46.077 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-ceph-argparse. 2026-03-24T02:12:46.083 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../12-python3-ceph-argparse_20.2.0-712-g70f8415b-1jammy_all.deb ... 2026-03-24T02:12:46.084 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-ceph-argparse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:46.094 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package liboath0:amd64. 2026-03-24T02:12:46.099 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-cephfs. 2026-03-24T02:12:46.100 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../21-liboath0_2.6.7-3ubuntu0.1_amd64.deb ... 2026-03-24T02:12:46.101 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-03-24T02:12:46.105 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../13-python3-cephfs_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:46.106 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-cephfs (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:46.118 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package libradosstriper1. 2026-03-24T02:12:46.123 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-ceph-common. 2026-03-24T02:12:46.124 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../22-libradosstriper1_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:46.125 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking libradosstriper1 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:46.130 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../14-python3-ceph-common_20.2.0-712-g70f8415b-1jammy_all.deb ... 2026-03-24T02:12:46.131 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-ceph-common (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:46.145 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package ceph-common. 2026-03-24T02:12:46.151 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../23-ceph-common_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:46.152 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking ceph-common (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:46.152 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-wcwidth. 2026-03-24T02:12:46.159 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../15-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2026-03-24T02:12:46.159 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2026-03-24T02:12:46.177 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-prettytable. 2026-03-24T02:12:46.183 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../16-python3-prettytable_2.5.0-2_all.deb ... 2026-03-24T02:12:46.184 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2026-03-24T02:12:46.199 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-rbd. 2026-03-24T02:12:46.205 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../17-python3-rbd_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:46.205 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-rbd (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:46.226 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package librdkafka1:amd64. 2026-03-24T02:12:46.233 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../18-librdkafka1_1.8.0-1build1_amd64.deb ... 2026-03-24T02:12:46.233 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2026-03-24T02:12:46.257 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package librgw2. 2026-03-24T02:12:46.263 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../19-librgw2_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:46.263 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking librgw2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:46.609 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-rgw. 2026-03-24T02:12:46.615 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../20-python3-rgw_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:46.616 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-rgw (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:46.633 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package ceph-base. 2026-03-24T02:12:46.634 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package liboath0:amd64. 2026-03-24T02:12:46.639 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../24-ceph-base_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:46.641 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../21-liboath0_2.6.7-3ubuntu0.1_amd64.deb ... 2026-03-24T02:12:46.641 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-03-24T02:12:46.644 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking ceph-base (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:46.657 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package libradosstriper1. 2026-03-24T02:12:46.663 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../22-libradosstriper1_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:46.664 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking libradosstriper1 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:46.831 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package ceph-common. 2026-03-24T02:12:46.838 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../23-ceph-common_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:46.839 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking ceph-common (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:46.843 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-jaraco.functools. 2026-03-24T02:12:46.850 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../25-python3-jaraco.functools_3.4.0-2_all.deb ... 2026-03-24T02:12:46.851 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2026-03-24T02:12:46.866 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-cheroot. 2026-03-24T02:12:46.872 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../26-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2026-03-24T02:12:46.873 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-03-24T02:12:46.893 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-jaraco.classes. 2026-03-24T02:12:46.899 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../27-python3-jaraco.classes_3.2.1-3_all.deb ... 2026-03-24T02:12:46.900 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2026-03-24T02:12:46.915 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-jaraco.text. 2026-03-24T02:12:46.920 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../28-python3-jaraco.text_3.6.0-2_all.deb ... 2026-03-24T02:12:46.921 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2026-03-24T02:12:46.936 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-jaraco.collections. 2026-03-24T02:12:46.942 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../29-python3-jaraco.collections_3.4.0-2_all.deb ... 2026-03-24T02:12:46.952 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2026-03-24T02:12:46.967 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-tempora. 2026-03-24T02:12:46.974 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../30-python3-tempora_4.1.2-1_all.deb ... 2026-03-24T02:12:46.975 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-tempora (4.1.2-1) ... 2026-03-24T02:12:46.991 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-portend. 2026-03-24T02:12:46.998 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../31-python3-portend_3.0.0-1_all.deb ... 2026-03-24T02:12:46.999 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-portend (3.0.0-1) ... 2026-03-24T02:12:47.014 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-zc.lockfile. 2026-03-24T02:12:47.023 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../32-python3-zc.lockfile_2.0-1_all.deb ... 2026-03-24T02:12:47.024 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2026-03-24T02:12:47.042 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-cherrypy3. 2026-03-24T02:12:47.050 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../33-python3-cherrypy3_18.6.1-4_all.deb ... 2026-03-24T02:12:47.051 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2026-03-24T02:12:47.083 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-natsort. 2026-03-24T02:12:47.089 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../34-python3-natsort_8.0.2-1_all.deb ... 2026-03-24T02:12:47.089 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-natsort (8.0.2-1) ... 2026-03-24T02:12:47.105 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2026-03-24T02:12:47.111 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../35-ceph-mgr-modules-core_20.2.0-712-g70f8415b-1jammy_all.deb ... 2026-03-24T02:12:47.118 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking ceph-mgr-modules-core (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:47.518 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2026-03-24T02:12:47.525 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../36-libsqlite3-mod-ceph_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:47.538 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking libsqlite3-mod-ceph (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:47.555 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package ceph-mgr. 2026-03-24T02:12:47.556 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package ceph-base. 2026-03-24T02:12:47.561 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../37-ceph-mgr_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:47.562 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking ceph-mgr (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:47.564 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../24-ceph-base_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:47.570 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking ceph-base (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:47.604 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package ceph-mon. 2026-03-24T02:12:47.610 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../38-ceph-mon_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:47.611 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking ceph-mon (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:47.720 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-jaraco.functools. 2026-03-24T02:12:47.724 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package libfuse2:amd64. 2026-03-24T02:12:47.725 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../25-python3-jaraco.functools_3.4.0-2_all.deb ... 2026-03-24T02:12:47.725 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2026-03-24T02:12:47.731 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../39-libfuse2_2.9.9-5ubuntu3_amd64.deb ... 2026-03-24T02:12:47.732 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-03-24T02:12:47.745 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-cheroot. 2026-03-24T02:12:47.751 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package ceph-osd. 2026-03-24T02:12:47.753 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../26-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2026-03-24T02:12:47.754 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-03-24T02:12:47.757 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../40-ceph-osd_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:47.759 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking ceph-osd (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:47.778 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-jaraco.classes. 2026-03-24T02:12:47.786 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../27-python3-jaraco.classes_3.2.1-3_all.deb ... 2026-03-24T02:12:47.786 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2026-03-24T02:12:47.807 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-jaraco.text. 2026-03-24T02:12:47.815 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../28-python3-jaraco.text_3.6.0-2_all.deb ... 2026-03-24T02:12:47.816 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2026-03-24T02:12:47.835 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-jaraco.collections. 2026-03-24T02:12:47.842 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../29-python3-jaraco.collections_3.4.0-2_all.deb ... 2026-03-24T02:12:47.843 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2026-03-24T02:12:47.873 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-tempora. 2026-03-24T02:12:47.880 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../30-python3-tempora_4.1.2-1_all.deb ... 2026-03-24T02:12:47.881 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-tempora (4.1.2-1) ... 2026-03-24T02:12:47.900 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-portend. 2026-03-24T02:12:47.907 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../31-python3-portend_3.0.0-1_all.deb ... 2026-03-24T02:12:47.908 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-portend (3.0.0-1) ... 2026-03-24T02:12:47.925 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-zc.lockfile. 2026-03-24T02:12:47.932 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../32-python3-zc.lockfile_2.0-1_all.deb ... 2026-03-24T02:12:47.933 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2026-03-24T02:12:48.060 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-cherrypy3. 2026-03-24T02:12:48.065 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../33-python3-cherrypy3_18.6.1-4_all.deb ... 2026-03-24T02:12:48.078 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2026-03-24T02:12:48.079 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package ceph. 2026-03-24T02:12:48.086 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../41-ceph_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:48.174 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking ceph (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:48.192 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package ceph-fuse. 2026-03-24T02:12:48.193 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-natsort. 2026-03-24T02:12:48.199 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../42-ceph-fuse_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:48.200 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking ceph-fuse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:48.201 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../34-python3-natsort_8.0.2-1_all.deb ... 2026-03-24T02:12:48.202 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-natsort (8.0.2-1) ... 2026-03-24T02:12:48.239 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2026-03-24T02:12:48.245 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package ceph-mds. 2026-03-24T02:12:48.245 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../35-ceph-mgr-modules-core_20.2.0-712-g70f8415b-1jammy_all.deb ... 2026-03-24T02:12:48.246 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking ceph-mgr-modules-core (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:48.252 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../43-ceph-mds_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:48.253 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking ceph-mds (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:48.291 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2026-03-24T02:12:48.298 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../36-libsqlite3-mod-ceph_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:48.299 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking libsqlite3-mod-ceph (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:48.303 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package cephadm. 2026-03-24T02:12:48.310 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../44-cephadm_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:48.311 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking cephadm (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:48.316 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package ceph-mgr. 2026-03-24T02:12:48.322 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../37-ceph-mgr_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:48.323 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking ceph-mgr (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:48.332 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-asyncssh. 2026-03-24T02:12:48.339 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../45-python3-asyncssh_2.5.0-1ubuntu0.1_all.deb ... 2026-03-24T02:12:48.339 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-03-24T02:12:48.351 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package ceph-mon. 2026-03-24T02:12:48.357 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../38-ceph-mon_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:48.358 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking ceph-mon (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:48.369 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2026-03-24T02:12:48.375 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../46-ceph-mgr-cephadm_20.2.0-712-g70f8415b-1jammy_all.deb ... 2026-03-24T02:12:48.376 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking ceph-mgr-cephadm (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:48.402 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-repoze.lru. 2026-03-24T02:12:48.408 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../47-python3-repoze.lru_0.7-2_all.deb ... 2026-03-24T02:12:48.409 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2026-03-24T02:12:48.471 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-routes. 2026-03-24T02:12:48.472 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package libfuse2:amd64. 2026-03-24T02:12:48.477 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../48-python3-routes_2.5.1-1ubuntu1_all.deb ... 2026-03-24T02:12:48.478 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../39-libfuse2_2.9.9-5ubuntu3_amd64.deb ... 2026-03-24T02:12:48.478 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2026-03-24T02:12:48.479 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-03-24T02:12:48.498 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package ceph-osd. 2026-03-24T02:12:48.505 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2026-03-24T02:12:48.505 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../40-ceph-osd_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:48.506 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking ceph-osd (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:48.511 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../49-ceph-mgr-dashboard_20.2.0-712-g70f8415b-1jammy_all.deb ... 2026-03-24T02:12:48.512 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking ceph-mgr-dashboard (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:48.813 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package ceph. 2026-03-24T02:12:48.820 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../41-ceph_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:48.821 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking ceph (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:48.837 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package ceph-fuse. 2026-03-24T02:12:48.843 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../42-ceph-fuse_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:48.844 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking ceph-fuse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:48.874 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package ceph-mds. 2026-03-24T02:12:48.882 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../43-ceph-mds_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:48.883 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking ceph-mds (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:48.950 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package cephadm. 2026-03-24T02:12:48.956 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../44-cephadm_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:48.957 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking cephadm (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:49.210 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-asyncssh. 2026-03-24T02:12:49.216 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../45-python3-asyncssh_2.5.0-1ubuntu0.1_all.deb ... 2026-03-24T02:12:49.217 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-03-24T02:12:49.244 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2026-03-24T02:12:49.251 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../46-ceph-mgr-cephadm_20.2.0-712-g70f8415b-1jammy_all.deb ... 2026-03-24T02:12:49.252 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking ceph-mgr-cephadm (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:49.282 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-repoze.lru. 2026-03-24T02:12:49.282 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2026-03-24T02:12:49.288 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../47-python3-repoze.lru_0.7-2_all.deb ... 2026-03-24T02:12:49.289 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../50-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2026-03-24T02:12:49.290 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2026-03-24T02:12:49.290 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-03-24T02:12:49.309 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-routes. 2026-03-24T02:12:49.316 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../48-python3-routes_2.5.1-1ubuntu1_all.deb ... 2026-03-24T02:12:49.317 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2026-03-24T02:12:49.366 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2026-03-24T02:12:49.369 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-joblib. 2026-03-24T02:12:49.372 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../49-ceph-mgr-dashboard_20.2.0-712-g70f8415b-1jammy_all.deb ... 2026-03-24T02:12:49.373 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking ceph-mgr-dashboard (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:49.373 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../51-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2026-03-24T02:12:49.374 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2026-03-24T02:12:49.417 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-threadpoolctl. 2026-03-24T02:12:49.421 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../52-python3-threadpoolctl_3.1.0-1_all.deb ... 2026-03-24T02:12:49.422 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2026-03-24T02:12:49.441 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-sklearn. 2026-03-24T02:12:49.447 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../53-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2026-03-24T02:12:49.448 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2026-03-24T02:12:49.585 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2026-03-24T02:12:49.591 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../54-ceph-mgr-diskprediction-local_20.2.0-712-g70f8415b-1jammy_all.deb ... 2026-03-24T02:12:49.592 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking ceph-mgr-diskprediction-local (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:50.128 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-cachetools. 2026-03-24T02:12:50.134 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../55-python3-cachetools_5.0.0-1_all.deb ... 2026-03-24T02:12:50.135 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2026-03-24T02:12:50.152 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-rsa. 2026-03-24T02:12:50.157 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../56-python3-rsa_4.8-1_all.deb ... 2026-03-24T02:12:50.158 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-rsa (4.8-1) ... 2026-03-24T02:12:50.180 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-google-auth. 2026-03-24T02:12:50.185 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../57-python3-google-auth_1.5.1-3_all.deb ... 2026-03-24T02:12:50.186 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2026-03-24T02:12:50.187 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2026-03-24T02:12:50.194 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../50-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2026-03-24T02:12:50.195 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-03-24T02:12:50.208 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-requests-oauthlib. 2026-03-24T02:12:50.216 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../58-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2026-03-24T02:12:50.216 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-03-24T02:12:50.256 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-websocket. 2026-03-24T02:12:50.260 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../59-python3-websocket_1.2.3-1_all.deb ... 2026-03-24T02:12:50.261 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-websocket (1.2.3-1) ... 2026-03-24T02:12:50.262 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-joblib. 2026-03-24T02:12:50.270 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../51-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2026-03-24T02:12:50.271 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2026-03-24T02:12:50.284 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-kubernetes. 2026-03-24T02:12:50.289 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../60-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2026-03-24T02:12:50.291 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2026-03-24T02:12:50.310 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-threadpoolctl. 2026-03-24T02:12:50.318 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../52-python3-threadpoolctl_3.1.0-1_all.deb ... 2026-03-24T02:12:50.319 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2026-03-24T02:12:50.338 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-sklearn. 2026-03-24T02:12:50.344 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../53-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2026-03-24T02:12:50.346 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2026-03-24T02:12:50.451 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2026-03-24T02:12:50.458 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../61-ceph-mgr-k8sevents_20.2.0-712-g70f8415b-1jammy_all.deb ... 2026-03-24T02:12:50.459 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking ceph-mgr-k8sevents (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:50.476 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2026-03-24T02:12:50.476 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package libonig5:amd64. 2026-03-24T02:12:50.481 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../62-libonig5_6.9.7.1-2build1_amd64.deb ... 2026-03-24T02:12:50.482 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../54-ceph-mgr-diskprediction-local_20.2.0-712-g70f8415b-1jammy_all.deb ... 2026-03-24T02:12:50.482 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2026-03-24T02:12:50.483 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking ceph-mgr-diskprediction-local (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:50.500 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package libjq1:amd64. 2026-03-24T02:12:50.506 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../63-libjq1_1.6-2.1ubuntu3.1_amd64.deb ... 2026-03-24T02:12:50.507 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-03-24T02:12:50.524 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package jq. 2026-03-24T02:12:50.530 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../64-jq_1.6-2.1ubuntu3.1_amd64.deb ... 2026-03-24T02:12:50.531 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking jq (1.6-2.1ubuntu3.1) ... 2026-03-24T02:12:50.548 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package socat. 2026-03-24T02:12:50.552 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../65-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2026-03-24T02:12:50.553 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2026-03-24T02:12:50.580 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package xmlstarlet. 2026-03-24T02:12:50.588 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../66-xmlstarlet_1.6.1-2.1_amd64.deb ... 2026-03-24T02:12:50.589 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2026-03-24T02:12:50.642 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package ceph-test. 2026-03-24T02:12:50.649 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../67-ceph-test_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:50.745 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking ceph-test (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:50.908 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-cachetools. 2026-03-24T02:12:50.915 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../55-python3-cachetools_5.0.0-1_all.deb ... 2026-03-24T02:12:50.916 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2026-03-24T02:12:50.936 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-rsa. 2026-03-24T02:12:50.942 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../56-python3-rsa_4.8-1_all.deb ... 2026-03-24T02:12:50.950 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-rsa (4.8-1) ... 2026-03-24T02:12:50.972 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-google-auth. 2026-03-24T02:12:50.977 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../57-python3-google-auth_1.5.1-3_all.deb ... 2026-03-24T02:12:50.978 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2026-03-24T02:12:51.001 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-requests-oauthlib. 2026-03-24T02:12:51.009 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../58-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2026-03-24T02:12:51.010 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-03-24T02:12:51.041 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-websocket. 2026-03-24T02:12:51.048 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../59-python3-websocket_1.2.3-1_all.deb ... 2026-03-24T02:12:51.049 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-websocket (1.2.3-1) ... 2026-03-24T02:12:51.071 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-kubernetes. 2026-03-24T02:12:51.077 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../60-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2026-03-24T02:12:51.078 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2026-03-24T02:12:51.231 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2026-03-24T02:12:51.239 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../61-ceph-mgr-k8sevents_20.2.0-712-g70f8415b-1jammy_all.deb ... 2026-03-24T02:12:51.240 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking ceph-mgr-k8sevents (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:51.257 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package libonig5:amd64. 2026-03-24T02:12:51.264 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../62-libonig5_6.9.7.1-2build1_amd64.deb ... 2026-03-24T02:12:51.265 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2026-03-24T02:12:51.285 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package libjq1:amd64. 2026-03-24T02:12:51.292 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../63-libjq1_1.6-2.1ubuntu3.1_amd64.deb ... 2026-03-24T02:12:51.293 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-03-24T02:12:51.322 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package jq. 2026-03-24T02:12:51.329 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../64-jq_1.6-2.1ubuntu3.1_amd64.deb ... 2026-03-24T02:12:51.330 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking jq (1.6-2.1ubuntu3.1) ... 2026-03-24T02:12:51.348 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package socat. 2026-03-24T02:12:51.355 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../65-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2026-03-24T02:12:51.356 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2026-03-24T02:12:51.383 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package xmlstarlet. 2026-03-24T02:12:51.391 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../66-xmlstarlet_1.6.1-2.1_amd64.deb ... 2026-03-24T02:12:51.392 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2026-03-24T02:12:51.440 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package ceph-test. 2026-03-24T02:12:51.449 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../67-ceph-test_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:51.450 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking ceph-test (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:53.116 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package ceph-volume. 2026-03-24T02:12:53.122 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../68-ceph-volume_20.2.0-712-g70f8415b-1jammy_all.deb ... 2026-03-24T02:12:53.123 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking ceph-volume (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:53.133 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package ceph-volume. 2026-03-24T02:12:53.140 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../68-ceph-volume_20.2.0-712-g70f8415b-1jammy_all.deb ... 2026-03-24T02:12:53.140 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking ceph-volume (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:53.156 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package libcephfs-daemon. 2026-03-24T02:12:53.163 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../69-libcephfs-daemon_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:53.164 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking libcephfs-daemon (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:53.183 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package libcephfs-daemon. 2026-03-24T02:12:53.191 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../69-libcephfs-daemon_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:53.192 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking libcephfs-daemon (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:53.196 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package libcephfs-proxy2. 2026-03-24T02:12:53.200 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../70-libcephfs-proxy2_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:53.201 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking libcephfs-proxy2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:53.209 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package libcephfs-proxy2. 2026-03-24T02:12:53.216 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../70-libcephfs-proxy2_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:53.217 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking libcephfs-proxy2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:53.217 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package libcephfs-dev. 2026-03-24T02:12:53.224 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../71-libcephfs-dev_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:53.225 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking libcephfs-dev (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:53.233 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package libcephfs-dev. 2026-03-24T02:12:53.242 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../71-libcephfs-dev_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:53.243 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking libcephfs-dev (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:53.245 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package nvme-cli. 2026-03-24T02:12:53.252 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../72-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2026-03-24T02:12:53.252 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2026-03-24T02:12:53.263 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package nvme-cli. 2026-03-24T02:12:53.271 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../72-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2026-03-24T02:12:53.274 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2026-03-24T02:12:53.296 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python-asyncssh-doc. 2026-03-24T02:12:53.302 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../73-python-asyncssh-doc_2.5.0-1ubuntu0.1_all.deb ... 2026-03-24T02:12:53.303 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-03-24T02:12:53.315 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python-asyncssh-doc. 2026-03-24T02:12:53.322 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../73-python-asyncssh-doc_2.5.0-1ubuntu0.1_all.deb ... 2026-03-24T02:12:53.323 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-03-24T02:12:53.354 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-iniconfig. 2026-03-24T02:12:53.361 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../74-python3-iniconfig_1.1.1-2_all.deb ... 2026-03-24T02:12:53.362 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2026-03-24T02:12:53.368 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-iniconfig. 2026-03-24T02:12:53.375 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../74-python3-iniconfig_1.1.1-2_all.deb ... 2026-03-24T02:12:53.375 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2026-03-24T02:12:53.378 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-pluggy. 2026-03-24T02:12:53.384 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../75-python3-pluggy_0.13.0-7.1_all.deb ... 2026-03-24T02:12:53.385 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2026-03-24T02:12:53.391 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-pluggy. 2026-03-24T02:12:53.398 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../75-python3-pluggy_0.13.0-7.1_all.deb ... 2026-03-24T02:12:53.406 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2026-03-24T02:12:53.415 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-psutil. 2026-03-24T02:12:53.422 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../76-python3-psutil_5.9.0-1build1_amd64.deb ... 2026-03-24T02:12:53.423 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2026-03-24T02:12:53.425 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-psutil. 2026-03-24T02:12:53.432 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../76-python3-psutil_5.9.0-1build1_amd64.deb ... 2026-03-24T02:12:53.433 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2026-03-24T02:12:53.446 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-py. 2026-03-24T02:12:53.453 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../77-python3-py_1.10.0-1_all.deb ... 2026-03-24T02:12:53.454 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-py (1.10.0-1) ... 2026-03-24T02:12:53.459 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-py. 2026-03-24T02:12:53.466 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../77-python3-py_1.10.0-1_all.deb ... 2026-03-24T02:12:53.467 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-py (1.10.0-1) ... 2026-03-24T02:12:53.478 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-pygments. 2026-03-24T02:12:53.485 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../78-python3-pygments_2.11.2+dfsg-2ubuntu0.1_all.deb ... 2026-03-24T02:12:53.485 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-03-24T02:12:53.493 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-pygments. 2026-03-24T02:12:53.500 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../78-python3-pygments_2.11.2+dfsg-2ubuntu0.1_all.deb ... 2026-03-24T02:12:53.501 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-03-24T02:12:53.554 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-toml. 2026-03-24T02:12:53.561 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../79-python3-toml_0.10.2-1_all.deb ... 2026-03-24T02:12:53.562 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-toml. 2026-03-24T02:12:53.562 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-toml (0.10.2-1) ... 2026-03-24T02:12:53.569 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../79-python3-toml_0.10.2-1_all.deb ... 2026-03-24T02:12:53.570 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-toml (0.10.2-1) ... 2026-03-24T02:12:53.581 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-pytest. 2026-03-24T02:12:53.587 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../80-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2026-03-24T02:12:53.588 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2026-03-24T02:12:53.590 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-pytest. 2026-03-24T02:12:53.598 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../80-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2026-03-24T02:12:53.599 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2026-03-24T02:12:53.640 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-simplejson. 2026-03-24T02:12:53.646 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-simplejson. 2026-03-24T02:12:53.647 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../81-python3-simplejson_3.17.6-1build1_amd64.deb ... 2026-03-24T02:12:53.648 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2026-03-24T02:12:53.653 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../81-python3-simplejson_3.17.6-1build1_amd64.deb ... 2026-03-24T02:12:53.654 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2026-03-24T02:12:53.670 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-webob. 2026-03-24T02:12:53.676 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../82-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2026-03-24T02:12:53.676 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-webob. 2026-03-24T02:12:53.677 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-03-24T02:12:53.684 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../82-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2026-03-24T02:12:53.684 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-03-24T02:12:53.696 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package qttranslations5-l10n. 2026-03-24T02:12:53.703 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../83-qttranslations5-l10n_5.15.3-1_all.deb ... 2026-03-24T02:12:53.704 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2026-03-24T02:12:53.705 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package qttranslations5-l10n. 2026-03-24T02:12:53.713 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../83-qttranslations5-l10n_5.15.3-1_all.deb ... 2026-03-24T02:12:53.714 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2026-03-24T02:12:53.840 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package radosgw. 2026-03-24T02:12:53.845 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package radosgw. 2026-03-24T02:12:53.847 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../84-radosgw_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:53.849 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking radosgw (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:53.854 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../84-radosgw_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:53.865 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking radosgw (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:54.436 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package rbd-fuse. 2026-03-24T02:12:54.443 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../85-rbd-fuse_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:54.444 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking rbd-fuse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:54.450 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package rbd-fuse. 2026-03-24T02:12:54.457 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../85-rbd-fuse_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:54.457 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking rbd-fuse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:54.463 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package rbd-mirror. 2026-03-24T02:12:54.470 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../86-rbd-mirror_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:54.471 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking rbd-mirror (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:54.476 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package rbd-mirror. 2026-03-24T02:12:54.482 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../86-rbd-mirror_20.2.0-712-g70f8415b-1jammy_amd64.deb ... 2026-03-24T02:12:54.482 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking rbd-mirror (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:54.571 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package smartmontools. 2026-03-24T02:12:54.572 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package smartmontools. 2026-03-24T02:12:54.578 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../87-smartmontools_7.2-1ubuntu0.1_amd64.deb ... 2026-03-24T02:12:54.578 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../87-smartmontools_7.2-1ubuntu0.1_amd64.deb ... 2026-03-24T02:12:54.586 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking smartmontools (7.2-1ubuntu0.1) ... 2026-03-24T02:12:54.588 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking smartmontools (7.2-1ubuntu0.1) ... 2026-03-24T02:12:54.628 INFO:teuthology.orchestra.run.vm07.stdout:Setting up smartmontools (7.2-1ubuntu0.1) ... 2026-03-24T02:12:54.632 INFO:teuthology.orchestra.run.vm06.stdout:Setting up smartmontools (7.2-1ubuntu0.1) ... 2026-03-24T02:12:54.893 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/smartd.service → /lib/systemd/system/smartmontools.service. 2026-03-24T02:12:54.894 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/smartmontools.service → /lib/systemd/system/smartmontools.service. 2026-03-24T02:12:54.899 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/smartd.service → /lib/systemd/system/smartmontools.service. 2026-03-24T02:12:54.899 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/smartmontools.service → /lib/systemd/system/smartmontools.service. 2026-03-24T02:12:55.259 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2026-03-24T02:12:55.282 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2026-03-24T02:12:55.330 INFO:teuthology.orchestra.run.vm06.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2026-03-24T02:12:55.332 INFO:teuthology.orchestra.run.vm06.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2026-03-24T02:12:55.357 INFO:teuthology.orchestra.run.vm07.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2026-03-24T02:12:55.360 INFO:teuthology.orchestra.run.vm07.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2026-03-24T02:12:55.401 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /lib/systemd/system/nvmefc-boot-connections.service. 2026-03-24T02:12:55.432 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /lib/systemd/system/nvmefc-boot-connections.service. 2026-03-24T02:12:55.635 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service → /lib/systemd/system/nvmf-autoconnect.service. 2026-03-24T02:12:55.658 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service → /lib/systemd/system/nvmf-autoconnect.service. 2026-03-24T02:12:56.025 INFO:teuthology.orchestra.run.vm06.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2026-03-24T02:12:56.035 INFO:teuthology.orchestra.run.vm07.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2026-03-24T02:12:56.043 INFO:teuthology.orchestra.run.vm06.stdout:Setting up cephadm (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:56.055 INFO:teuthology.orchestra.run.vm07.stdout:Setting up cephadm (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:56.090 INFO:teuthology.orchestra.run.vm06.stdout:Adding system user cephadm....done 2026-03-24T02:12:56.100 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2026-03-24T02:12:56.101 INFO:teuthology.orchestra.run.vm07.stdout:Adding system user cephadm....done 2026-03-24T02:12:56.110 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2026-03-24T02:12:56.174 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-03-24T02:12:56.176 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2026-03-24T02:12:56.183 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-03-24T02:12:56.185 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2026-03-24T02:12:56.248 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-repoze.lru (0.7-2) ... 2026-03-24T02:12:56.257 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-repoze.lru (0.7-2) ... 2026-03-24T02:12:56.327 INFO:teuthology.orchestra.run.vm06.stdout:Setting up liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-03-24T02:12:56.327 INFO:teuthology.orchestra.run.vm07.stdout:Setting up liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-03-24T02:12:56.329 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-py (1.10.0-1) ... 2026-03-24T02:12:56.329 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-py (1.10.0-1) ... 2026-03-24T02:12:56.423 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2026-03-24T02:12:56.423 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2026-03-24T02:12:56.555 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-cachetools (5.0.0-1) ... 2026-03-24T02:12:56.555 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-cachetools (5.0.0-1) ... 2026-03-24T02:12:56.634 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2026-03-24T02:12:56.634 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2026-03-24T02:12:56.717 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-ceph-argparse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:56.717 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-ceph-argparse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:56.790 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-03-24T02:12:56.793 INFO:teuthology.orchestra.run.vm06.stdout:Setting up libnbd0 (1.10.5-1) ... 2026-03-24T02:12:56.793 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-03-24T02:12:56.795 INFO:teuthology.orchestra.run.vm06.stdout:Setting up libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-03-24T02:12:56.795 INFO:teuthology.orchestra.run.vm07.stdout:Setting up libnbd0 (1.10.5-1) ... 2026-03-24T02:12:56.798 INFO:teuthology.orchestra.run.vm06.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-03-24T02:12:56.798 INFO:teuthology.orchestra.run.vm07.stdout:Setting up libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-03-24T02:12:56.801 INFO:teuthology.orchestra.run.vm07.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-03-24T02:12:56.801 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2026-03-24T02:12:56.803 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2026-03-24T02:12:56.930 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-natsort (8.0.2-1) ... 2026-03-24T02:12:56.934 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-natsort (8.0.2-1) ... 2026-03-24T02:12:57.062 INFO:teuthology.orchestra.run.vm06.stdout:Setting up libcephfs-proxy2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:57.063 INFO:teuthology.orchestra.run.vm07.stdout:Setting up libcephfs-proxy2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:57.230 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2026-03-24T02:12:57.232 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2026-03-24T02:12:57.327 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2026-03-24T02:12:57.327 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2026-03-24T02:12:57.424 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-03-24T02:12:57.424 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-03-24T02:12:57.725 INFO:teuthology.orchestra.run.vm07.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2026-03-24T02:12:57.731 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2026-03-24T02:12:57.737 INFO:teuthology.orchestra.run.vm06.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2026-03-24T02:12:57.740 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2026-03-24T02:12:57.833 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-03-24T02:12:57.840 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-03-24T02:12:57.979 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-03-24T02:12:57.994 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-03-24T02:12:58.074 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2026-03-24T02:12:58.089 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2026-03-24T02:12:58.147 INFO:teuthology.orchestra.run.vm07.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2026-03-24T02:12:58.149 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-ceph-common (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:58.162 INFO:teuthology.orchestra.run.vm06.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2026-03-24T02:12:58.165 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-ceph-common (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:12:58.256 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2026-03-24T02:12:58.274 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2026-03-24T02:12:58.848 INFO:teuthology.orchestra.run.vm07.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:12:58.854 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-toml (0.10.2-1) ... 2026-03-24T02:12:58.873 INFO:teuthology.orchestra.run.vm06.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:12:58.879 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-toml (0.10.2-1) ... 2026-03-24T02:12:58.935 INFO:teuthology.orchestra.run.vm07.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2026-03-24T02:12:58.937 INFO:teuthology.orchestra.run.vm07.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2026-03-24T02:12:58.940 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2026-03-24T02:12:58.957 INFO:teuthology.orchestra.run.vm06.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2026-03-24T02:12:58.960 INFO:teuthology.orchestra.run.vm06.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2026-03-24T02:12:58.962 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2026-03-24T02:12:59.015 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2026-03-24T02:12:59.043 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2026-03-24T02:12:59.088 INFO:teuthology.orchestra.run.vm07.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:12:59.091 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-rsa (4.8-1) ... 2026-03-24T02:12:59.116 INFO:teuthology.orchestra.run.vm06.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:12:59.119 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-rsa (4.8-1) ... 2026-03-24T02:12:59.173 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-tempora (4.1.2-1) ... 2026-03-24T02:12:59.198 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-tempora (4.1.2-1) ... 2026-03-24T02:12:59.247 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-prettytable (2.5.0-2) ... 2026-03-24T02:12:59.273 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-prettytable (2.5.0-2) ... 2026-03-24T02:12:59.332 INFO:teuthology.orchestra.run.vm07.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-03-24T02:12:59.334 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-websocket (1.2.3-1) ... 2026-03-24T02:12:59.359 INFO:teuthology.orchestra.run.vm06.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-03-24T02:12:59.362 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-websocket (1.2.3-1) ... 2026-03-24T02:12:59.418 INFO:teuthology.orchestra.run.vm07.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2026-03-24T02:12:59.422 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-03-24T02:12:59.446 INFO:teuthology.orchestra.run.vm06.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2026-03-24T02:12:59.449 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-03-24T02:12:59.503 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-03-24T02:12:59.525 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-03-24T02:12:59.598 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2026-03-24T02:12:59.615 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2026-03-24T02:12:59.669 INFO:teuthology.orchestra.run.vm07.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-03-24T02:12:59.671 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2026-03-24T02:12:59.690 INFO:teuthology.orchestra.run.vm06.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-03-24T02:12:59.693 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2026-03-24T02:12:59.819 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-portend (3.0.0-1) ... 2026-03-24T02:12:59.848 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-portend (3.0.0-1) ... 2026-03-24T02:12:59.891 INFO:teuthology.orchestra.run.vm07.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:12:59.894 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-google-auth (1.5.1-3) ... 2026-03-24T02:12:59.926 INFO:teuthology.orchestra.run.vm06.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:12:59.928 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-google-auth (1.5.1-3) ... 2026-03-24T02:12:59.984 INFO:teuthology.orchestra.run.vm07.stdout:Setting up jq (1.6-2.1ubuntu3.1) ... 2026-03-24T02:12:59.986 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2026-03-24T02:13:00.021 INFO:teuthology.orchestra.run.vm06.stdout:Setting up jq (1.6-2.1ubuntu3.1) ... 2026-03-24T02:13:00.024 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2026-03-24T02:13:00.127 INFO:teuthology.orchestra.run.vm07.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-03-24T02:13:00.130 INFO:teuthology.orchestra.run.vm07.stdout:Setting up librados2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:00.132 INFO:teuthology.orchestra.run.vm07.stdout:Setting up librgw2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:00.135 INFO:teuthology.orchestra.run.vm07.stdout:Setting up libsqlite3-mod-ceph (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:00.137 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2026-03-24T02:13:00.162 INFO:teuthology.orchestra.run.vm06.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-03-24T02:13:00.164 INFO:teuthology.orchestra.run.vm06.stdout:Setting up librados2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:00.167 INFO:teuthology.orchestra.run.vm06.stdout:Setting up librgw2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:00.169 INFO:teuthology.orchestra.run.vm06.stdout:Setting up libsqlite3-mod-ceph (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:00.172 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2026-03-24T02:13:00.713 INFO:teuthology.orchestra.run.vm07.stdout:Setting up libcephfs2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:00.716 INFO:teuthology.orchestra.run.vm07.stdout:Setting up libradosstriper1 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:00.719 INFO:teuthology.orchestra.run.vm07.stdout:Setting up librbd1 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:00.721 INFO:teuthology.orchestra.run.vm07.stdout:Setting up ceph-mgr-modules-core (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:00.723 INFO:teuthology.orchestra.run.vm07.stdout:Setting up ceph-fuse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:00.779 INFO:teuthology.orchestra.run.vm06.stdout:Setting up libcephfs2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:00.781 INFO:teuthology.orchestra.run.vm06.stdout:Setting up libradosstriper1 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:00.784 INFO:teuthology.orchestra.run.vm06.stdout:Setting up librbd1 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:00.786 INFO:teuthology.orchestra.run.vm06.stdout:Setting up ceph-mgr-modules-core (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:00.789 INFO:teuthology.orchestra.run.vm06.stdout:Setting up ceph-fuse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:00.789 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-03-24T02:13:00.789 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-03-24T02:13:00.855 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-03-24T02:13:00.855 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-03-24T02:13:01.166 INFO:teuthology.orchestra.run.vm07.stdout:Setting up libcephfs-dev (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:01.172 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-rados (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:01.191 INFO:teuthology.orchestra.run.vm07.stdout:Setting up libcephfs-daemon (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:01.194 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-rbd (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:01.196 INFO:teuthology.orchestra.run.vm07.stdout:Setting up rbd-fuse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:01.199 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-rgw (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:01.201 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-cephfs (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:01.206 INFO:teuthology.orchestra.run.vm07.stdout:Setting up ceph-common (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:01.234 INFO:teuthology.orchestra.run.vm06.stdout:Setting up libcephfs-dev (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:01.237 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-rados (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:01.239 INFO:teuthology.orchestra.run.vm06.stdout:Setting up libcephfs-daemon (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:01.242 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-rbd (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:01.242 INFO:teuthology.orchestra.run.vm07.stdout:Adding group ceph....done 2026-03-24T02:13:01.245 INFO:teuthology.orchestra.run.vm06.stdout:Setting up rbd-fuse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:01.247 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-rgw (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:01.250 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-cephfs (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:01.253 INFO:teuthology.orchestra.run.vm06.stdout:Setting up ceph-common (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:01.280 INFO:teuthology.orchestra.run.vm07.stdout:Adding system user ceph....done 2026-03-24T02:13:01.288 INFO:teuthology.orchestra.run.vm06.stdout:Adding group ceph....done 2026-03-24T02:13:01.290 INFO:teuthology.orchestra.run.vm07.stdout:Setting system user ceph properties....done 2026-03-24T02:13:01.294 INFO:teuthology.orchestra.run.vm07.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2026-03-24T02:13:01.325 INFO:teuthology.orchestra.run.vm06.stdout:Adding system user ceph....done 2026-03-24T02:13:01.343 INFO:teuthology.orchestra.run.vm06.stdout:Setting system user ceph properties....done 2026-03-24T02:13:01.348 INFO:teuthology.orchestra.run.vm06.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2026-03-24T02:13:01.366 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /lib/systemd/system/ceph.target. 2026-03-24T02:13:01.422 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /lib/systemd/system/ceph.target. 2026-03-24T02:13:01.612 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service → /lib/systemd/system/rbdmap.service. 2026-03-24T02:13:01.668 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service → /lib/systemd/system/rbdmap.service. 2026-03-24T02:13:02.016 INFO:teuthology.orchestra.run.vm07.stdout:Setting up ceph-test (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:02.018 INFO:teuthology.orchestra.run.vm07.stdout:Setting up radosgw (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:02.071 INFO:teuthology.orchestra.run.vm06.stdout:Setting up ceph-test (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:02.074 INFO:teuthology.orchestra.run.vm06.stdout:Setting up radosgw (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:02.312 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-03-24T02:13:02.312 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-03-24T02:13:02.323 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-03-24T02:13:02.323 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-03-24T02:13:02.700 INFO:teuthology.orchestra.run.vm07.stdout:Setting up ceph-base (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:02.715 INFO:teuthology.orchestra.run.vm06.stdout:Setting up ceph-base (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:02.796 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /lib/systemd/system/ceph-crash.service. 2026-03-24T02:13:02.812 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /lib/systemd/system/ceph-crash.service. 2026-03-24T02:13:03.189 INFO:teuthology.orchestra.run.vm06.stdout:Setting up ceph-mds (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:03.218 INFO:teuthology.orchestra.run.vm07.stdout:Setting up ceph-mds (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:03.258 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-03-24T02:13:03.258 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-03-24T02:13:03.288 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-03-24T02:13:03.288 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-03-24T02:13:03.654 INFO:teuthology.orchestra.run.vm06.stdout:Setting up rbd-mirror (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:03.676 INFO:teuthology.orchestra.run.vm07.stdout:Setting up rbd-mirror (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:03.731 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-03-24T02:13:03.731 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-03-24T02:13:03.753 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-03-24T02:13:03.753 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-03-24T02:13:04.145 INFO:teuthology.orchestra.run.vm06.stdout:Setting up ceph-mgr (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:04.180 INFO:teuthology.orchestra.run.vm07.stdout:Setting up ceph-mgr (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:04.213 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-03-24T02:13:04.213 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-03-24T02:13:04.250 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-03-24T02:13:04.251 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-03-24T02:13:04.598 INFO:teuthology.orchestra.run.vm06.stdout:Setting up ceph-osd (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:04.642 INFO:teuthology.orchestra.run.vm07.stdout:Setting up ceph-osd (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:04.692 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-03-24T02:13:04.692 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-03-24T02:13:04.738 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-03-24T02:13:04.738 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-03-24T02:13:05.069 INFO:teuthology.orchestra.run.vm06.stdout:Setting up ceph-mgr-k8sevents (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:05.072 INFO:teuthology.orchestra.run.vm06.stdout:Setting up ceph-mgr-diskprediction-local (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:05.086 INFO:teuthology.orchestra.run.vm06.stdout:Setting up ceph-mon (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:05.124 INFO:teuthology.orchestra.run.vm07.stdout:Setting up ceph-mgr-k8sevents (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:05.131 INFO:teuthology.orchestra.run.vm07.stdout:Setting up ceph-mgr-diskprediction-local (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:05.153 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-03-24T02:13:05.153 INFO:teuthology.orchestra.run.vm06.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-03-24T02:13:05.158 INFO:teuthology.orchestra.run.vm07.stdout:Setting up ceph-mon (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:05.226 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-03-24T02:13:05.226 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-03-24T02:13:05.562 INFO:teuthology.orchestra.run.vm06.stdout:Setting up ceph-mgr-cephadm (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:05.575 INFO:teuthology.orchestra.run.vm06.stdout:Setting up ceph (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:05.577 INFO:teuthology.orchestra.run.vm06.stdout:Setting up ceph-mgr-dashboard (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:05.590 INFO:teuthology.orchestra.run.vm06.stdout:Setting up ceph-volume (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:05.629 INFO:teuthology.orchestra.run.vm07.stdout:Setting up ceph-mgr-cephadm (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:05.643 INFO:teuthology.orchestra.run.vm07.stdout:Setting up ceph (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:05.646 INFO:teuthology.orchestra.run.vm07.stdout:Setting up ceph-mgr-dashboard (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:05.659 INFO:teuthology.orchestra.run.vm07.stdout:Setting up ceph-volume (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:13:05.720 INFO:teuthology.orchestra.run.vm06.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-03-24T02:13:05.784 INFO:teuthology.orchestra.run.vm07.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-03-24T02:13:05.803 INFO:teuthology.orchestra.run.vm06.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-03-24T02:13:05.878 INFO:teuthology.orchestra.run.vm07.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-03-24T02:13:06.131 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:06.131 INFO:teuthology.orchestra.run.vm06.stdout:Running kernel seems to be up-to-date. 2026-03-24T02:13:06.131 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:06.131 INFO:teuthology.orchestra.run.vm06.stdout:Services to be restarted: 2026-03-24T02:13:06.134 INFO:teuthology.orchestra.run.vm06.stdout: systemctl restart apache-htcacheclean.service 2026-03-24T02:13:06.139 INFO:teuthology.orchestra.run.vm06.stdout: systemctl restart rsyslog.service 2026-03-24T02:13:06.143 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:06.143 INFO:teuthology.orchestra.run.vm06.stdout:Service restarts being deferred: 2026-03-24T02:13:06.143 INFO:teuthology.orchestra.run.vm06.stdout: systemctl restart networkd-dispatcher.service 2026-03-24T02:13:06.143 INFO:teuthology.orchestra.run.vm06.stdout: systemctl restart unattended-upgrades.service 2026-03-24T02:13:06.143 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:06.143 INFO:teuthology.orchestra.run.vm06.stdout:No containers need to be restarted. 2026-03-24T02:13:06.143 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:06.143 INFO:teuthology.orchestra.run.vm06.stdout:No user sessions are running outdated binaries. 2026-03-24T02:13:06.143 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:06.143 INFO:teuthology.orchestra.run.vm06.stdout:No VM guests are running outdated hypervisor (qemu) binaries on this host. 2026-03-24T02:13:06.197 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:06.197 INFO:teuthology.orchestra.run.vm07.stdout:Running kernel seems to be up-to-date. 2026-03-24T02:13:06.197 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:06.197 INFO:teuthology.orchestra.run.vm07.stdout:Services to be restarted: 2026-03-24T02:13:06.200 INFO:teuthology.orchestra.run.vm07.stdout: systemctl restart apache-htcacheclean.service 2026-03-24T02:13:06.205 INFO:teuthology.orchestra.run.vm07.stdout: systemctl restart rsyslog.service 2026-03-24T02:13:06.208 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:06.208 INFO:teuthology.orchestra.run.vm07.stdout:Service restarts being deferred: 2026-03-24T02:13:06.208 INFO:teuthology.orchestra.run.vm07.stdout: systemctl restart networkd-dispatcher.service 2026-03-24T02:13:06.208 INFO:teuthology.orchestra.run.vm07.stdout: systemctl restart unattended-upgrades.service 2026-03-24T02:13:06.208 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:06.208 INFO:teuthology.orchestra.run.vm07.stdout:No containers need to be restarted. 2026-03-24T02:13:06.208 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:06.208 INFO:teuthology.orchestra.run.vm07.stdout:No user sessions are running outdated binaries. 2026-03-24T02:13:06.208 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:06.208 INFO:teuthology.orchestra.run.vm07.stdout:No VM guests are running outdated hypervisor (qemu) binaries on this host. 2026-03-24T02:13:07.073 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:13:07.076 DEBUG:teuthology.orchestra.run.vm06:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install python3-jmespath python3-xmltodict s3cmd 2026-03-24T02:13:07.078 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:13:07.081 DEBUG:teuthology.orchestra.run.vm07:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install python3-jmespath python3-xmltodict s3cmd 2026-03-24T02:13:07.155 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:13:07.159 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:13:07.350 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:13:07.351 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:13:07.361 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:13:07.362 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:13:07.516 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:13:07.516 INFO:teuthology.orchestra.run.vm07.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libpmemobj1 2026-03-24T02:13:07.516 INFO:teuthology.orchestra.run.vm07.stdout: libsgutils2-2 sg3-utils sg3-utils-udev 2026-03-24T02:13:07.516 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:13:07.524 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:13:07.524 INFO:teuthology.orchestra.run.vm06.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libpmemobj1 2026-03-24T02:13:07.524 INFO:teuthology.orchestra.run.vm06.stdout: libsgutils2-2 sg3-utils sg3-utils-udev 2026-03-24T02:13:07.524 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:13:07.533 INFO:teuthology.orchestra.run.vm07.stdout:The following NEW packages will be installed: 2026-03-24T02:13:07.533 INFO:teuthology.orchestra.run.vm07.stdout: python3-jmespath python3-xmltodict s3cmd 2026-03-24T02:13:07.540 INFO:teuthology.orchestra.run.vm06.stdout:The following NEW packages will be installed: 2026-03-24T02:13:07.540 INFO:teuthology.orchestra.run.vm06.stdout: python3-jmespath python3-xmltodict s3cmd 2026-03-24T02:13:07.624 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 3 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:13:07.624 INFO:teuthology.orchestra.run.vm07.stdout:Need to get 155 kB of archives. 2026-03-24T02:13:07.624 INFO:teuthology.orchestra.run.vm07.stdout:After this operation, 678 kB of additional disk space will be used. 2026-03-24T02:13:07.624 INFO:teuthology.orchestra.run.vm07.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jmespath all 0.10.0-1 [21.7 kB] 2026-03-24T02:13:07.709 INFO:teuthology.orchestra.run.vm07.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-xmltodict all 0.12.0-2 [12.6 kB] 2026-03-24T02:13:07.714 INFO:teuthology.orchestra.run.vm07.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy/universe amd64 s3cmd all 2.2.0-1 [120 kB] 2026-03-24T02:13:07.779 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 3 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:13:07.779 INFO:teuthology.orchestra.run.vm06.stdout:Need to get 155 kB of archives. 2026-03-24T02:13:07.779 INFO:teuthology.orchestra.run.vm06.stdout:After this operation, 678 kB of additional disk space will be used. 2026-03-24T02:13:07.779 INFO:teuthology.orchestra.run.vm06.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jmespath all 0.10.0-1 [21.7 kB] 2026-03-24T02:13:08.008 INFO:teuthology.orchestra.run.vm07.stdout:Fetched 155 kB in 0s (568 kB/s) 2026-03-24T02:13:08.025 INFO:teuthology.orchestra.run.vm06.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-xmltodict all 0.12.0-2 [12.6 kB] 2026-03-24T02:13:08.050 INFO:teuthology.orchestra.run.vm06.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy/universe amd64 s3cmd all 2.2.0-1 [120 kB] 2026-03-24T02:13:08.243 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-jmespath. 2026-03-24T02:13:08.273 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 126089 files and directories currently installed.) 2026-03-24T02:13:08.275 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../python3-jmespath_0.10.0-1_all.deb ... 2026-03-24T02:13:08.276 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-jmespath (0.10.0-1) ... 2026-03-24T02:13:08.295 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package python3-xmltodict. 2026-03-24T02:13:08.302 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../python3-xmltodict_0.12.0-2_all.deb ... 2026-03-24T02:13:08.303 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking python3-xmltodict (0.12.0-2) ... 2026-03-24T02:13:08.320 INFO:teuthology.orchestra.run.vm07.stdout:Selecting previously unselected package s3cmd. 2026-03-24T02:13:08.328 INFO:teuthology.orchestra.run.vm07.stdout:Preparing to unpack .../archives/s3cmd_2.2.0-1_all.deb ... 2026-03-24T02:13:08.329 INFO:teuthology.orchestra.run.vm07.stdout:Unpacking s3cmd (2.2.0-1) ... 2026-03-24T02:13:08.362 INFO:teuthology.orchestra.run.vm07.stdout:Setting up s3cmd (2.2.0-1) ... 2026-03-24T02:13:08.483 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-xmltodict (0.12.0-2) ... 2026-03-24T02:13:08.496 INFO:teuthology.orchestra.run.vm06.stdout:Fetched 155 kB in 1s (205 kB/s) 2026-03-24T02:13:08.697 INFO:teuthology.orchestra.run.vm07.stdout:Setting up python3-jmespath (0.10.0-1) ... 2026-03-24T02:13:09.059 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-jmespath. 2026-03-24T02:13:09.089 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 126089 files and directories currently installed.) 2026-03-24T02:13:09.090 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../python3-jmespath_0.10.0-1_all.deb ... 2026-03-24T02:13:09.146 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-jmespath (0.10.0-1) ... 2026-03-24T02:13:09.201 INFO:teuthology.orchestra.run.vm07.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-03-24T02:13:09.227 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package python3-xmltodict. 2026-03-24T02:13:09.232 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../python3-xmltodict_0.12.0-2_all.deb ... 2026-03-24T02:13:09.232 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking python3-xmltodict (0.12.0-2) ... 2026-03-24T02:13:09.248 INFO:teuthology.orchestra.run.vm06.stdout:Selecting previously unselected package s3cmd. 2026-03-24T02:13:09.254 INFO:teuthology.orchestra.run.vm06.stdout:Preparing to unpack .../archives/s3cmd_2.2.0-1_all.deb ... 2026-03-24T02:13:09.254 INFO:teuthology.orchestra.run.vm06.stdout:Unpacking s3cmd (2.2.0-1) ... 2026-03-24T02:13:09.286 INFO:teuthology.orchestra.run.vm06.stdout:Setting up s3cmd (2.2.0-1) ... 2026-03-24T02:13:09.384 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-xmltodict (0.12.0-2) ... 2026-03-24T02:13:09.459 INFO:teuthology.orchestra.run.vm06.stdout:Setting up python3-jmespath (0.10.0-1) ... 2026-03-24T02:13:09.536 INFO:teuthology.orchestra.run.vm06.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-03-24T02:13:09.551 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:09.551 INFO:teuthology.orchestra.run.vm07.stdout:Running kernel seems to be up-to-date. 2026-03-24T02:13:09.551 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:09.551 INFO:teuthology.orchestra.run.vm07.stdout:Services to be restarted: 2026-03-24T02:13:09.554 INFO:teuthology.orchestra.run.vm07.stdout: systemctl restart apache-htcacheclean.service 2026-03-24T02:13:09.561 INFO:teuthology.orchestra.run.vm07.stdout: systemctl restart rsyslog.service 2026-03-24T02:13:09.564 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:09.564 INFO:teuthology.orchestra.run.vm07.stdout:Service restarts being deferred: 2026-03-24T02:13:09.564 INFO:teuthology.orchestra.run.vm07.stdout: systemctl restart networkd-dispatcher.service 2026-03-24T02:13:09.564 INFO:teuthology.orchestra.run.vm07.stdout: systemctl restart unattended-upgrades.service 2026-03-24T02:13:09.564 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:09.564 INFO:teuthology.orchestra.run.vm07.stdout:No containers need to be restarted. 2026-03-24T02:13:09.564 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:09.564 INFO:teuthology.orchestra.run.vm07.stdout:No user sessions are running outdated binaries. 2026-03-24T02:13:09.564 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:09.564 INFO:teuthology.orchestra.run.vm07.stdout:No VM guests are running outdated hypervisor (qemu) binaries on this host. 2026-03-24T02:13:09.863 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:09.864 INFO:teuthology.orchestra.run.vm06.stdout:Running kernel seems to be up-to-date. 2026-03-24T02:13:09.864 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:09.864 INFO:teuthology.orchestra.run.vm06.stdout:Services to be restarted: 2026-03-24T02:13:09.866 INFO:teuthology.orchestra.run.vm06.stdout: systemctl restart apache-htcacheclean.service 2026-03-24T02:13:09.871 INFO:teuthology.orchestra.run.vm06.stdout: systemctl restart rsyslog.service 2026-03-24T02:13:09.874 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:09.874 INFO:teuthology.orchestra.run.vm06.stdout:Service restarts being deferred: 2026-03-24T02:13:09.874 INFO:teuthology.orchestra.run.vm06.stdout: systemctl restart networkd-dispatcher.service 2026-03-24T02:13:09.874 INFO:teuthology.orchestra.run.vm06.stdout: systemctl restart unattended-upgrades.service 2026-03-24T02:13:09.874 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:09.874 INFO:teuthology.orchestra.run.vm06.stdout:No containers need to be restarted. 2026-03-24T02:13:09.874 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:09.874 INFO:teuthology.orchestra.run.vm06.stdout:No user sessions are running outdated binaries. 2026-03-24T02:13:09.874 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:09.874 INFO:teuthology.orchestra.run.vm06.stdout:No VM guests are running outdated hypervisor (qemu) binaries on this host. 2026-03-24T02:13:10.565 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:13:10.569 DEBUG:teuthology.parallel:result is None 2026-03-24T02:13:10.901 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:13:10.906 DEBUG:teuthology.parallel:result is None 2026-03-24T02:13:10.906 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=70f8415b300f041766fa27faf7d5472699e32388 2026-03-24T02:13:11.538 DEBUG:teuthology.orchestra.run.vm06:> dpkg-query -W -f '${Version}' ceph 2026-03-24T02:13:11.547 INFO:teuthology.orchestra.run.vm06.stdout:20.2.0-712-g70f8415b-1jammy 2026-03-24T02:13:11.547 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-712-g70f8415b-1jammy 2026-03-24T02:13:11.547 INFO:teuthology.task.install:The correct ceph version 20.2.0-712-g70f8415b-1jammy is installed. 2026-03-24T02:13:11.548 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=70f8415b300f041766fa27faf7d5472699e32388 2026-03-24T02:13:12.214 DEBUG:teuthology.orchestra.run.vm07:> dpkg-query -W -f '${Version}' ceph 2026-03-24T02:13:12.223 INFO:teuthology.orchestra.run.vm07.stdout:20.2.0-712-g70f8415b-1jammy 2026-03-24T02:13:12.223 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-712-g70f8415b-1jammy 2026-03-24T02:13:12.224 INFO:teuthology.task.install:The correct ceph version 20.2.0-712-g70f8415b-1jammy is installed. 2026-03-24T02:13:12.224 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2026-03-24T02:13:12.224 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:12.225 DEBUG:teuthology.orchestra.run.vm06:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-03-24T02:13:12.234 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:12.234 DEBUG:teuthology.orchestra.run.vm07:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-03-24T02:13:12.275 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2026-03-24T02:13:12.275 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:12.275 DEBUG:teuthology.orchestra.run.vm06:> sudo dd of=/usr/bin/daemon-helper 2026-03-24T02:13:12.284 DEBUG:teuthology.orchestra.run.vm06:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-03-24T02:13:12.336 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:12.336 DEBUG:teuthology.orchestra.run.vm07:> sudo dd of=/usr/bin/daemon-helper 2026-03-24T02:13:12.343 DEBUG:teuthology.orchestra.run.vm07:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-03-24T02:13:12.395 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2026-03-24T02:13:12.395 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:12.395 DEBUG:teuthology.orchestra.run.vm06:> sudo dd of=/usr/bin/adjust-ulimits 2026-03-24T02:13:12.404 DEBUG:teuthology.orchestra.run.vm06:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-03-24T02:13:12.456 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:12.456 DEBUG:teuthology.orchestra.run.vm07:> sudo dd of=/usr/bin/adjust-ulimits 2026-03-24T02:13:12.464 DEBUG:teuthology.orchestra.run.vm07:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-03-24T02:13:12.515 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2026-03-24T02:13:12.515 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:12.515 DEBUG:teuthology.orchestra.run.vm06:> sudo dd of=/usr/bin/stdin-killer 2026-03-24T02:13:12.523 DEBUG:teuthology.orchestra.run.vm06:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-03-24T02:13:12.571 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:12.571 DEBUG:teuthology.orchestra.run.vm07:> sudo dd of=/usr/bin/stdin-killer 2026-03-24T02:13:12.579 DEBUG:teuthology.orchestra.run.vm07:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-03-24T02:13:12.630 INFO:teuthology.run_tasks:Running task ceph... 2026-03-24T02:13:12.672 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2026-03-24T02:13:12.672 DEBUG:teuthology.orchestra.run.vm06:> sudo chmod 777 /var/log/ceph 2026-03-24T02:13:12.674 DEBUG:teuthology.orchestra.run.vm07:> sudo chmod 777 /var/log/ceph 2026-03-24T02:13:12.682 INFO:tasks.ceph:Disabling ceph logrotate... 2026-03-24T02:13:12.682 DEBUG:teuthology.orchestra.run.vm06:> sudo rm -f -- /etc/logrotate.d/ceph 2026-03-24T02:13:12.724 DEBUG:teuthology.orchestra.run.vm07:> sudo rm -f -- /etc/logrotate.d/ceph 2026-03-24T02:13:12.732 INFO:tasks.ceph:Creating extra log directories... 2026-03-24T02:13:12.732 DEBUG:teuthology.orchestra.run.vm06:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-03-24T02:13:12.772 DEBUG:teuthology.orchestra.run.vm07:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-03-24T02:13:12.790 INFO:tasks.ceph:Creating ceph cluster cluster1... 2026-03-24T02:13:12.790 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-03-24T02:13:12.790 INFO:tasks.ceph:ctx.config {'archive_path': '/archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3539', 'branch': 'tentacle', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-journal-stress-workunit}', 'email': None, 'first_in_suite': False, 'flavor': 'default', 'job_id': '3539', 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'vps', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps', 'no_nested_subset': False, 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': 'tentacle'}, 'ansible.cephlab': {'branch': 'main', 'repo': 'https://github.com/kshtsk/ceph-cm-ansible.git', 'skip_tags': 'nagios,monitoring-scripts,hostname,pubkeys,zap,sudoers,kerberos,ntp-client,resolvconf,cpan,nfs', 'vars': {'logical_volumes': {'lv_1': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_2': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_3': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_4': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}}, 'timezone': 'UTC', 'volume_groups': {'vg_nvme': {'pvs': '/dev/vdb,/dev/vdc,/dev/vdd,/dev/vde'}}}}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'cephadm': {'cephadm_binary_url': 'https://download.ceph.com/rpm-20.2.0/el9/noarch/cephadm'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}, 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'tt-tentacle', 'sha1': '0392f78529848ec72469e8e431875cb98d3a5fb4'}}, 'owner': 'kyr', 'priority': 1000, 'repo': 'https://github.com/ceph/ceph.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 3051, 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'sleep_before_teardown': 0, 'subset': '1/128', 'suite': 'rbd', 'suite_branch': 'tt-tentacle', 'suite_path': '/home/teuthos/src/github.com_kshtsk_ceph_0392f78529848ec72469e8e431875cb98d3a5fb4/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/kshtsk/ceph.git', 'suite_sha1': '0392f78529848ec72469e8e431875cb98d3a5fb4', 'targets': {'vm06.local': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFMdzVQ4qWHGmJfQDRxmfOu8MjX2Enb0f1yGF9w/uaSFB9MTnpB4aag7G6CcWBNqpWlI9EJwSapJNyvZr8HZKHc=', 'vm07.local': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK5C8C/vVG/hbJJiioODjMgoQKxU7Psh6pK8go4Z7ooenLW0HVrniuzl2OdiCD1ftad+WLA/q04SzvgeWCQAHV4='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'pool', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'journal', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'clyso-debian-13', 'teuthology_repo': 'https://github.com/clyso/teuthology', 'teuthology_sha1': '1c580df7a9c7c2aadc272da296344fd99f27c444', 'timestamp': '2026-03-20_22:04:26', 'tube': 'vps', 'user': 'kyr', 'verbose': False, 'worker_log': '/home/teuthos/.teuthology/dispatcher/dispatcher.vps.2366871'} 2026-03-24T02:13:12.790 DEBUG:teuthology.orchestra.run.vm06:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-03-24T02:13:12.820 DEBUG:teuthology.orchestra.run.vm07:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-03-24T02:13:12.827 DEBUG:teuthology.orchestra.run.vm06:> sudo install -d -m0777 -- /var/run/ceph 2026-03-24T02:13:12.864 DEBUG:teuthology.orchestra.run.vm07:> sudo install -d -m0777 -- /var/run/ceph 2026-03-24T02:13:12.875 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:12.875 DEBUG:teuthology.orchestra.run.vm06:> dd if=/scratch_devs of=/dev/stdout 2026-03-24T02:13:12.923 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-03-24T02:13:12.923 DEBUG:teuthology.orchestra.run.vm06:> stat /dev/vg_nvme/lv_1 2026-03-24T02:13:12.967 INFO:teuthology.orchestra.run.vm06.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-03-24T02:13:12.967 INFO:teuthology.orchestra.run.vm06.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-24T02:13:12.967 INFO:teuthology.orchestra.run.vm06.stdout:Device: 5h/5d Inode: 782 Links: 1 2026-03-24T02:13:12.967 INFO:teuthology.orchestra.run.vm06.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-24T02:13:12.967 INFO:teuthology.orchestra.run.vm06.stdout:Access: 2026-03-24 02:12:12.304922694 +0000 2026-03-24T02:13:12.967 INFO:teuthology.orchestra.run.vm06.stdout:Modify: 2026-03-24 02:12:12.180922694 +0000 2026-03-24T02:13:12.967 INFO:teuthology.orchestra.run.vm06.stdout:Change: 2026-03-24 02:12:12.180922694 +0000 2026-03-24T02:13:12.967 INFO:teuthology.orchestra.run.vm06.stdout: Birth: - 2026-03-24T02:13:12.967 DEBUG:teuthology.orchestra.run.vm06:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-03-24T02:13:13.015 INFO:teuthology.orchestra.run.vm06.stderr:1+0 records in 2026-03-24T02:13:13.016 INFO:teuthology.orchestra.run.vm06.stderr:1+0 records out 2026-03-24T02:13:13.016 INFO:teuthology.orchestra.run.vm06.stderr:512 bytes copied, 0.000175629 s, 2.9 MB/s 2026-03-24T02:13:13.016 DEBUG:teuthology.orchestra.run.vm06:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-03-24T02:13:13.064 DEBUG:teuthology.orchestra.run.vm06:> stat /dev/vg_nvme/lv_2 2026-03-24T02:13:13.112 INFO:teuthology.orchestra.run.vm06.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-03-24T02:13:13.112 INFO:teuthology.orchestra.run.vm06.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-24T02:13:13.112 INFO:teuthology.orchestra.run.vm06.stdout:Device: 5h/5d Inode: 812 Links: 1 2026-03-24T02:13:13.112 INFO:teuthology.orchestra.run.vm06.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-24T02:13:13.112 INFO:teuthology.orchestra.run.vm06.stdout:Access: 2026-03-24 02:12:12.452922694 +0000 2026-03-24T02:13:13.112 INFO:teuthology.orchestra.run.vm06.stdout:Modify: 2026-03-24 02:12:12.448922694 +0000 2026-03-24T02:13:13.112 INFO:teuthology.orchestra.run.vm06.stdout:Change: 2026-03-24 02:12:12.448922694 +0000 2026-03-24T02:13:13.112 INFO:teuthology.orchestra.run.vm06.stdout: Birth: - 2026-03-24T02:13:13.112 DEBUG:teuthology.orchestra.run.vm06:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-03-24T02:13:13.166 INFO:teuthology.orchestra.run.vm06.stderr:1+0 records in 2026-03-24T02:13:13.166 INFO:teuthology.orchestra.run.vm06.stderr:1+0 records out 2026-03-24T02:13:13.166 INFO:teuthology.orchestra.run.vm06.stderr:512 bytes copied, 0.000163937 s, 3.1 MB/s 2026-03-24T02:13:13.167 DEBUG:teuthology.orchestra.run.vm06:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-03-24T02:13:13.212 DEBUG:teuthology.orchestra.run.vm06:> stat /dev/vg_nvme/lv_3 2026-03-24T02:13:13.259 INFO:teuthology.orchestra.run.vm06.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-03-24T02:13:13.259 INFO:teuthology.orchestra.run.vm06.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-24T02:13:13.259 INFO:teuthology.orchestra.run.vm06.stdout:Device: 5h/5d Inode: 846 Links: 1 2026-03-24T02:13:13.259 INFO:teuthology.orchestra.run.vm06.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-24T02:13:13.259 INFO:teuthology.orchestra.run.vm06.stdout:Access: 2026-03-24 02:12:12.948922694 +0000 2026-03-24T02:13:13.259 INFO:teuthology.orchestra.run.vm06.stdout:Modify: 2026-03-24 02:12:12.820922694 +0000 2026-03-24T02:13:13.259 INFO:teuthology.orchestra.run.vm06.stdout:Change: 2026-03-24 02:12:12.820922694 +0000 2026-03-24T02:13:13.259 INFO:teuthology.orchestra.run.vm06.stdout: Birth: - 2026-03-24T02:13:13.259 DEBUG:teuthology.orchestra.run.vm06:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-03-24T02:13:13.307 INFO:teuthology.orchestra.run.vm06.stderr:1+0 records in 2026-03-24T02:13:13.307 INFO:teuthology.orchestra.run.vm06.stderr:1+0 records out 2026-03-24T02:13:13.307 INFO:teuthology.orchestra.run.vm06.stderr:512 bytes copied, 0.000169938 s, 3.0 MB/s 2026-03-24T02:13:13.307 DEBUG:teuthology.orchestra.run.vm06:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-03-24T02:13:13.356 DEBUG:teuthology.orchestra.run.vm06:> stat /dev/vg_nvme/lv_4 2026-03-24T02:13:13.404 INFO:teuthology.orchestra.run.vm06.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-03-24T02:13:13.405 INFO:teuthology.orchestra.run.vm06.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-24T02:13:13.405 INFO:teuthology.orchestra.run.vm06.stdout:Device: 5h/5d Inode: 873 Links: 1 2026-03-24T02:13:13.405 INFO:teuthology.orchestra.run.vm06.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-24T02:13:13.405 INFO:teuthology.orchestra.run.vm06.stdout:Access: 2026-03-24 02:12:16.800922694 +0000 2026-03-24T02:13:13.405 INFO:teuthology.orchestra.run.vm06.stdout:Modify: 2026-03-24 02:12:13.104922694 +0000 2026-03-24T02:13:13.405 INFO:teuthology.orchestra.run.vm06.stdout:Change: 2026-03-24 02:12:13.104922694 +0000 2026-03-24T02:13:13.405 INFO:teuthology.orchestra.run.vm06.stdout: Birth: - 2026-03-24T02:13:13.405 DEBUG:teuthology.orchestra.run.vm06:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-03-24T02:13:13.454 INFO:teuthology.orchestra.run.vm06.stderr:1+0 records in 2026-03-24T02:13:13.454 INFO:teuthology.orchestra.run.vm06.stderr:1+0 records out 2026-03-24T02:13:13.454 INFO:teuthology.orchestra.run.vm06.stderr:512 bytes copied, 0.000186579 s, 2.7 MB/s 2026-03-24T02:13:13.455 DEBUG:teuthology.orchestra.run.vm06:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-03-24T02:13:13.499 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-24T02:13:13.499 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@vm06.local'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-03-24T02:13:13.499 INFO:tasks.ceph:Generating config... 2026-03-24T02:13:13.500 INFO:tasks.ceph:[client] debug rbd = 20 2026-03-24T02:13:13.500 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-03-24T02:13:13.500 INFO:tasks.ceph:[client] log to stderr = False 2026-03-24T02:13:13.500 INFO:tasks.ceph:[client] rbd default features = 125 2026-03-24T02:13:13.500 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2026-03-24T02:13:13.500 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:13.500 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:13.500 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:13.500 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:13.500 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:13.500 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:13.500 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:13.500 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:13.500 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-03-24T02:13:13.500 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-03-24T02:13:13.500 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-03-24T02:13:13.500 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-03-24T02:13:13.500 INFO:tasks.ceph:[mgr] debug ms = 1 2026-03-24T02:13:13.500 INFO:tasks.ceph:[mon] debug mon = 20 2026-03-24T02:13:13.500 INFO:tasks.ceph:[mon] debug ms = 1 2026-03-24T02:13:13.500 INFO:tasks.ceph:[mon] debug paxos = 20 2026-03-24T02:13:13.500 INFO:tasks.ceph:[osd] bluefs allocator = hybrid 2026-03-24T02:13:13.500 INFO:tasks.ceph:[osd] bluestore allocator = hybrid 2026-03-24T02:13:13.500 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-03-24T02:13:13.500 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-03-24T02:13:13.500 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-03-24T02:13:13.500 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-03-24T02:13:13.500 INFO:tasks.ceph:[osd] debug ms = 1 2026-03-24T02:13:13.500 INFO:tasks.ceph:[osd] debug osd = 20 2026-03-24T02:13:13.500 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-03-24T02:13:13.500 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-03-24T02:13:13.500 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-03-24T02:13:13.500 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-03-24T02:13:13.500 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-03-24T02:13:13.500 INFO:tasks.ceph:[osd] osd mclock iops capacity threshold hdd = 49000 2026-03-24T02:13:13.500 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-03-24T02:13:13.500 INFO:tasks.ceph:Setting up cluster1.mon.a... 2026-03-24T02:13:13.500 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2026-03-24T02:13:13.555 INFO:teuthology.orchestra.run.vm06.stdout:creating /etc/ceph/cluster1.keyring 2026-03-24T02:13:13.557 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2026-03-24T02:13:13.617 DEBUG:teuthology.orchestra.run.vm06:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-03-24T02:13:13.666 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '192.168.123.106')] 2026-03-24T02:13:13.666 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '192.168.123.106', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-03-24T02:13:13.666 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:13.666 DEBUG:teuthology.orchestra.run.vm06:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-03-24T02:13:13.710 DEBUG:teuthology.orchestra.run.vm06:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 192.168.123.106 --print /home/ubuntu/cephtest/cluster1.monmap 2026-03-24T02:13:13.768 INFO:teuthology.orchestra.run.vm06.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2026-03-24T02:13:13.768 INFO:teuthology.orchestra.run.vm06.stdout:monmaptool: generated fsid 91306d06-3c56-4223-8b51-5365acf96311 2026-03-24T02:13:13.768 INFO:teuthology.orchestra.run.vm06.stdout:setting min_mon_release = tentacle 2026-03-24T02:13:13.768 INFO:teuthology.orchestra.run.vm06.stdout:epoch 0 2026-03-24T02:13:13.768 INFO:teuthology.orchestra.run.vm06.stdout:fsid 91306d06-3c56-4223-8b51-5365acf96311 2026-03-24T02:13:13.768 INFO:teuthology.orchestra.run.vm06.stdout:last_changed 2026-03-24T02:13:13.769544+0000 2026-03-24T02:13:13.768 INFO:teuthology.orchestra.run.vm06.stdout:created 2026-03-24T02:13:13.769544+0000 2026-03-24T02:13:13.768 INFO:teuthology.orchestra.run.vm06.stdout:min_mon_release 20 (tentacle) 2026-03-24T02:13:13.768 INFO:teuthology.orchestra.run.vm06.stdout:election_strategy: 1 2026-03-24T02:13:13.768 INFO:teuthology.orchestra.run.vm06.stdout:0: [v2:192.168.123.106:3300/0,v1:192.168.123.106:6789/0] mon.a 2026-03-24T02:13:13.768 INFO:teuthology.orchestra.run.vm06.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2026-03-24T02:13:13.769 DEBUG:teuthology.orchestra.run.vm06:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-03-24T02:13:13.814 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID 91306d06-3c56-4223-8b51-5365acf96311... 2026-03-24T02:13:13.815 DEBUG:teuthology.orchestra.run.vm06:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-03-24T02:13:13.856 DEBUG:teuthology.orchestra.run.vm07:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout:[global] 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: chdir = "" 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: auth supported = cephx 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: filestore xattr use omap = true 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: mon clock drift allowed = 1.000 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: osd crush chooseleaf type = 0 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: auth debug = true 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: ms die on old message = true 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: ms die on bug = true 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: mon max pg per osd = 10000 # >= luminous 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: mon pg warn max object skew = 0 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: # disable pg_autoscaler by default for new pools 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: osd_pool_default_pg_autoscale_mode = off 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: osd pool default size = 2 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: mon osd allow primary affinity = true 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: mon osd allow pg remap = true 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: mon warn on legacy crush tunables = false 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: mon warn on crush straw calc version zero = false 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: mon warn on no sortbitwise = false 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: mon warn on osd down out interval zero = false 2026-03-24T02:13:13.870 INFO:teuthology.orchestra.run.vm06.stdout: mon warn on too few osds = false 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: mon_warn_on_pool_no_redundancy = false 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: mon_allow_pool_size_one = true 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd default data pool replay window = 5 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: mon allow pool delete = true 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: mon cluster log file level = debug 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: debug asserts on shutdown = true 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: mon health detail to clog = false 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: mon host = 192.168.123.106 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: mon client directed command retry = 5 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: mon warn on pool no app = False 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: ms inject socket failures = 5000 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: fsid = 91306d06-3c56-4223-8b51-5365acf96311 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout:[osd] 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd journal size = 100 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd scrub load threshold = 5.0 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd scrub max interval = 600 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd mclock profile = high_recovery_ops 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd mclock skip benchmark = true 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd recover clone overlap = true 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd recovery max chunk = 1048576 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd debug shutdown = true 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd debug op order = true 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd debug verify stray on activate = true 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd debug trim objects = true 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd open classes on start = true 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd debug pg log writeout = true 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd deep scrub update digest min age = 30 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd map max advance = 10 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: journal zero on create = true 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: filestore ondisk finisher threads = 3 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: filestore apply finisher threads = 3 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: bdev debug aio = true 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd debug misdirected ops = true 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: bluefs allocator = hybrid 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: bluestore allocator = hybrid 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: bluestore block size = 96636764160 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: bluestore fsck on mount = True 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: debug bluefs = 1/20 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: debug bluestore = 1/20 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: debug ms = 1 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: debug osd = 20 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: debug rocksdb = 4/10 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: mon osd backfillfull_ratio = 0.85 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: mon osd full ratio = 0.9 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: mon osd nearfull ratio = 0.8 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd failsafe full ratio = 0.95 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd mclock iops capacity threshold hdd = 49000 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: osd objectstore = bluestore 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.871 INFO:teuthology.orchestra.run.vm06.stdout:[mgr] 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: debug ms = 1 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: debug mgr = 20 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: debug mon = 20 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: debug auth = 20 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: mon reweight min pgs per osd = 4 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: mon reweight min bytes per osd = 10 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: mgr/telemetry/nag = false 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout:[mon] 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: debug ms = 1 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: debug mon = 20 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: debug paxos = 20 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: debug auth = 20 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: mon data avail warn = 5 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: mon mgr mkfs grace = 240 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: mon reweight min pgs per osd = 4 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: mon osd reporter subtree level = osd 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: mon osd prime pg temp = true 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: mon reweight min bytes per osd = 10 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: auth mon ticket ttl = 660 # 11m 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: auth service ticket ttl = 240 # 4m 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: # don't complain about insecure global_id in the test suite 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: # 1m isn't quite enough 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: mon_down_mkfs_grace = 2m 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: mon_warn_on_filestore_osds = false 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout:[client] 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: rgw cache enabled = true 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: rgw enable ops log = true 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: rgw enable usage log = true 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: debug rbd = 20 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: debug rbd_mirror = 15 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: log to stderr = False 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: rbd default features = 125 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: rbd mirror image policy type = simple 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout:[mon.a] 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout:[client.mirror.0] 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout:[client.mirror.1] 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout:[client.mirror.2] 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout:[client.mirror.3] 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:13.872 INFO:teuthology.orchestra.run.vm06.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:13.873 INFO:teuthology.orchestra.run.vm07.stdout:[global] 2026-03-24T02:13:13.873 INFO:teuthology.orchestra.run.vm07.stdout: chdir = "" 2026-03-24T02:13:13.873 INFO:teuthology.orchestra.run.vm07.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-03-24T02:13:13.873 INFO:teuthology.orchestra.run.vm07.stdout: auth supported = cephx 2026-03-24T02:13:13.873 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: filestore xattr use omap = true 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon clock drift allowed = 1.000 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: osd crush chooseleaf type = 0 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: auth debug = true 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: ms die on old message = true 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: ms die on bug = true 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon max pg per osd = 10000 # >= luminous 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon pg warn max object skew = 0 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: # disable pg_autoscaler by default for new pools 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: osd_pool_default_pg_autoscale_mode = off 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: osd pool default size = 2 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon osd allow primary affinity = true 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon osd allow pg remap = true 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on legacy crush tunables = false 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on crush straw calc version zero = false 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on no sortbitwise = false 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on osd down out interval zero = false 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on too few osds = false 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_pool_no_redundancy = false 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon_allow_pool_size_one = true 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: osd default data pool replay window = 5 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon allow pool delete = true 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon cluster log file level = debug 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: debug asserts on shutdown = true 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon health detail to clog = false 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon host = 192.168.123.106 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon client directed command retry = 5 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on pool no app = False 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: ms inject socket failures = 5000 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: fsid = 91306d06-3c56-4223-8b51-5365acf96311 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout:[osd] 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: osd journal size = 100 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: osd scrub load threshold = 5.0 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: osd scrub max interval = 600 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: osd mclock profile = high_recovery_ops 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: osd mclock skip benchmark = true 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: osd recover clone overlap = true 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: osd recovery max chunk = 1048576 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.874 INFO:teuthology.orchestra.run.vm07.stdout: osd debug shutdown = true 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: osd debug op order = true 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: osd debug verify stray on activate = true 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: osd debug trim objects = true 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: osd open classes on start = true 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: osd debug pg log writeout = true 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: osd deep scrub update digest min age = 30 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: osd map max advance = 10 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: journal zero on create = true 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: filestore ondisk finisher threads = 3 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: filestore apply finisher threads = 3 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: bdev debug aio = true 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: osd debug misdirected ops = true 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: bluefs allocator = hybrid 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: bluestore allocator = hybrid 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: bluestore block size = 96636764160 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: bluestore fsck on mount = True 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: debug bluefs = 1/20 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: debug bluestore = 1/20 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: debug ms = 1 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: debug osd = 20 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: debug rocksdb = 4/10 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: mon osd backfillfull_ratio = 0.85 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: mon osd full ratio = 0.9 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: mon osd nearfull ratio = 0.8 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: osd failsafe full ratio = 0.95 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: osd mclock iops capacity threshold hdd = 49000 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: osd objectstore = bluestore 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout:[mgr] 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: debug ms = 1 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: debug mgr = 20 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: debug mon = 20 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: debug auth = 20 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: mon reweight min pgs per osd = 4 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: mon reweight min bytes per osd = 10 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: mgr/telemetry/nag = false 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout:[mon] 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: debug ms = 1 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: debug mon = 20 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: debug paxos = 20 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: debug auth = 20 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: mon data avail warn = 5 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: mon mgr mkfs grace = 240 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: mon reweight min pgs per osd = 4 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: mon osd reporter subtree level = osd 2026-03-24T02:13:13.875 INFO:teuthology.orchestra.run.vm07.stdout: mon osd prime pg temp = true 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: mon reweight min bytes per osd = 10 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: auth mon ticket ttl = 660 # 11m 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: auth service ticket ttl = 240 # 4m 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: # don't complain about insecure global_id in the test suite 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: # 1m isn't quite enough 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: mon_down_mkfs_grace = 2m 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_filestore_osds = false 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout:[client] 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: rgw cache enabled = true 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: rgw enable ops log = true 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: rgw enable usage log = true 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: debug rbd = 20 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: debug rbd_mirror = 15 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: log to stderr = False 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: rbd default features = 125 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: rbd mirror image policy type = simple 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout:[mon.a] 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.0] 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.1] 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.2] 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.3] 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:13.876 INFO:teuthology.orchestra.run.vm07.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:13.879 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2026-03-24T02:13:13.880 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2026-03-24T02:13:13.942 INFO:tasks.ceph:Copying monmap to all nodes... 2026-03-24T02:13:13.945 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:13.945 DEBUG:teuthology.orchestra.run.vm06:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2026-03-24T02:13:13.986 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:14.013 DEBUG:teuthology.orchestra.run.vm06:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2026-03-24T02:13:14.030 INFO:tasks.ceph:Sending monmap to node ubuntu@vm06.local 2026-03-24T02:13:14.030 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:14.030 DEBUG:teuthology.orchestra.run.vm06:> sudo dd of=/etc/ceph/cluster1.keyring 2026-03-24T02:13:14.030 DEBUG:teuthology.orchestra.run.vm06:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-03-24T02:13:14.082 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:14.083 DEBUG:teuthology.orchestra.run.vm06:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-03-24T02:13:14.126 INFO:tasks.ceph:Sending monmap to node ubuntu@vm07.local 2026-03-24T02:13:14.126 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:14.126 DEBUG:teuthology.orchestra.run.vm07:> sudo dd of=/etc/ceph/cluster1.keyring 2026-03-24T02:13:14.126 DEBUG:teuthology.orchestra.run.vm07:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-03-24T02:13:14.138 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:14.145 DEBUG:teuthology.orchestra.run.vm07:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-03-24T02:13:14.182 INFO:tasks.ceph:Setting up mon nodes... 2026-03-24T02:13:14.182 INFO:tasks.ceph:Setting up mgr nodes... 2026-03-24T02:13:14.182 DEBUG:teuthology.orchestra.run.vm06:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2026-03-24T02:13:14.204 INFO:teuthology.orchestra.run.vm06.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2026-03-24T02:13:14.206 INFO:tasks.ceph:Setting up mds nodes... 2026-03-24T02:13:14.206 INFO:tasks.ceph_client:Setting up client nodes... 2026-03-24T02:13:14.206 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2026-03-24T02:13:14.265 INFO:teuthology.orchestra.run.vm06.stdout:creating /etc/ceph/cluster1.client.0.keyring 2026-03-24T02:13:14.272 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2026-03-24T02:13:14.292 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2026-03-24T02:13:14.299 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2026-03-24T02:13:14.361 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2026-03-24T02:13:14.368 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2026-03-24T02:13:14.430 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2026-03-24T02:13:14.438 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2026-03-24T02:13:14.498 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2026-03-24T02:13:14.505 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2026-03-24T02:13:14.565 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2026-03-24T02:13:14.573 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2026-03-24T02:13:14.633 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2026-03-24T02:13:14.641 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2026-03-24T02:13:14.701 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2026-03-24T02:13:14.758 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2026-03-24T02:13:14.820 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2026-03-24T02:13:14.828 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-03-24T02:13:14.828 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@vm06.local'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-03-24T02:13:14.828 DEBUG:teuthology.orchestra.run.vm06:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2026-03-24T02:13:14.838 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-24T02:13:14.838 INFO:tasks.ceph:role: cluster1.osd.0 2026-03-24T02:13:14.838 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@vm06.local 2026-03-24T02:13:14.838 DEBUG:teuthology.orchestra.run.vm06:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-03-24T02:13:14.889 INFO:teuthology.orchestra.run.vm06.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=1310464 blks 2026-03-24T02:13:14.889 INFO:teuthology.orchestra.run.vm06.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-24T02:13:14.889 INFO:teuthology.orchestra.run.vm06.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-24T02:13:14.889 INFO:teuthology.orchestra.run.vm06.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-03-24T02:13:14.889 INFO:teuthology.orchestra.run.vm06.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-24T02:13:14.889 INFO:teuthology.orchestra.run.vm06.stdout: = sunit=0 swidth=0 blks 2026-03-24T02:13:14.889 INFO:teuthology.orchestra.run.vm06.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-24T02:13:14.889 INFO:teuthology.orchestra.run.vm06.stdout:log =internal log bsize=4096 blocks=2560, version=2 2026-03-24T02:13:14.889 INFO:teuthology.orchestra.run.vm06.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-24T02:13:14.889 INFO:teuthology.orchestra.run.vm06.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-24T02:13:14.893 INFO:teuthology.orchestra.run.vm06.stdout:Discarding blocks...Done. 2026-03-24T02:13:14.895 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@vm06.local -o noatime 2026-03-24T02:13:14.895 DEBUG:teuthology.orchestra.run.vm06:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2026-03-24T02:13:14.990 DEBUG:teuthology.orchestra.run.vm06:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2026-03-24T02:13:14.996 INFO:teuthology.orchestra.run.vm06.stderr:sudo: /sbin/restorecon: command not found 2026-03-24T02:13:14.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-24T02:13:14.996 DEBUG:teuthology.orchestra.run.vm06:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2026-03-24T02:13:15.046 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-24T02:13:15.046 INFO:tasks.ceph:role: cluster1.osd.1 2026-03-24T02:13:15.046 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@vm06.local 2026-03-24T02:13:15.046 DEBUG:teuthology.orchestra.run.vm06:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-03-24T02:13:15.094 INFO:teuthology.orchestra.run.vm06.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=1310464 blks 2026-03-24T02:13:15.094 INFO:teuthology.orchestra.run.vm06.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-24T02:13:15.094 INFO:teuthology.orchestra.run.vm06.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-24T02:13:15.094 INFO:teuthology.orchestra.run.vm06.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-03-24T02:13:15.094 INFO:teuthology.orchestra.run.vm06.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-24T02:13:15.095 INFO:teuthology.orchestra.run.vm06.stdout: = sunit=0 swidth=0 blks 2026-03-24T02:13:15.095 INFO:teuthology.orchestra.run.vm06.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-24T02:13:15.095 INFO:teuthology.orchestra.run.vm06.stdout:log =internal log bsize=4096 blocks=2560, version=2 2026-03-24T02:13:15.095 INFO:teuthology.orchestra.run.vm06.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-24T02:13:15.095 INFO:teuthology.orchestra.run.vm06.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-24T02:13:15.099 INFO:teuthology.orchestra.run.vm06.stdout:Discarding blocks...Done. 2026-03-24T02:13:15.100 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@vm06.local -o noatime 2026-03-24T02:13:15.100 DEBUG:teuthology.orchestra.run.vm06:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2026-03-24T02:13:15.154 DEBUG:teuthology.orchestra.run.vm06:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2026-03-24T02:13:15.201 INFO:teuthology.orchestra.run.vm06.stderr:sudo: /sbin/restorecon: command not found 2026-03-24T02:13:15.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-24T02:13:15.201 DEBUG:teuthology.orchestra.run.vm06:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2026-03-24T02:13:15.250 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-24T02:13:15.251 INFO:tasks.ceph:role: cluster1.osd.2 2026-03-24T02:13:15.251 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@vm06.local 2026-03-24T02:13:15.251 DEBUG:teuthology.orchestra.run.vm06:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-03-24T02:13:15.299 INFO:teuthology.orchestra.run.vm06.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=1310464 blks 2026-03-24T02:13:15.299 INFO:teuthology.orchestra.run.vm06.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-24T02:13:15.299 INFO:teuthology.orchestra.run.vm06.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-24T02:13:15.299 INFO:teuthology.orchestra.run.vm06.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-03-24T02:13:15.299 INFO:teuthology.orchestra.run.vm06.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-24T02:13:15.299 INFO:teuthology.orchestra.run.vm06.stdout: = sunit=0 swidth=0 blks 2026-03-24T02:13:15.299 INFO:teuthology.orchestra.run.vm06.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-24T02:13:15.299 INFO:teuthology.orchestra.run.vm06.stdout:log =internal log bsize=4096 blocks=2560, version=2 2026-03-24T02:13:15.299 INFO:teuthology.orchestra.run.vm06.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-24T02:13:15.299 INFO:teuthology.orchestra.run.vm06.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-24T02:13:15.303 INFO:teuthology.orchestra.run.vm06.stdout:Discarding blocks...Done. 2026-03-24T02:13:15.305 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@vm06.local -o noatime 2026-03-24T02:13:15.305 DEBUG:teuthology.orchestra.run.vm06:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2026-03-24T02:13:15.359 DEBUG:teuthology.orchestra.run.vm06:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2026-03-24T02:13:15.409 INFO:teuthology.orchestra.run.vm06.stderr:sudo: /sbin/restorecon: command not found 2026-03-24T02:13:15.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-24T02:13:15.410 DEBUG:teuthology.orchestra.run.vm06:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-03-24T02:13:15.475 INFO:teuthology.orchestra.run.vm06.stderr:2026-03-24T02:13:15.470+0000 7f18e4c8ca40 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2026-03-24T02:13:15.476 INFO:teuthology.orchestra.run.vm06.stderr:2026-03-24T02:13:15.470+0000 7f18e4c8ca40 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2026-03-24T02:13:15.476 INFO:teuthology.orchestra.run.vm06.stderr:2026-03-24T02:13:15.474+0000 7f18e4c8ca40 -1 bdev(0x560da2c99800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2026-03-24T02:13:15.476 INFO:teuthology.orchestra.run.vm06.stderr:2026-03-24T02:13:15.474+0000 7f18e4c8ca40 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2026-03-24T02:13:16.359 DEBUG:teuthology.orchestra.run.vm06:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2026-03-24T02:13:16.408 DEBUG:teuthology.orchestra.run.vm06:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-03-24T02:13:16.472 INFO:teuthology.orchestra.run.vm06.stderr:2026-03-24T02:13:16.470+0000 7f156a902a40 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2026-03-24T02:13:16.472 INFO:teuthology.orchestra.run.vm06.stderr:2026-03-24T02:13:16.470+0000 7f156a902a40 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2026-03-24T02:13:16.472 INFO:teuthology.orchestra.run.vm06.stderr:2026-03-24T02:13:16.470+0000 7f156a902a40 -1 bdev(0x55df96a8b800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2026-03-24T02:13:16.472 INFO:teuthology.orchestra.run.vm06.stderr:2026-03-24T02:13:16.470+0000 7f156a902a40 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2026-03-24T02:13:17.355 DEBUG:teuthology.orchestra.run.vm06:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2026-03-24T02:13:17.407 DEBUG:teuthology.orchestra.run.vm06:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-03-24T02:13:17.479 INFO:teuthology.orchestra.run.vm06.stderr:2026-03-24T02:13:17.474+0000 7f91589e9a40 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2026-03-24T02:13:17.480 INFO:teuthology.orchestra.run.vm06.stderr:2026-03-24T02:13:17.478+0000 7f91589e9a40 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2026-03-24T02:13:17.480 INFO:teuthology.orchestra.run.vm06.stderr:2026-03-24T02:13:17.478+0000 7f91589e9a40 -1 bdev(0x555e9cd41800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2026-03-24T02:13:17.480 INFO:teuthology.orchestra.run.vm06.stderr:2026-03-24T02:13:17.478+0000 7f91589e9a40 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2026-03-24T02:13:18.375 DEBUG:teuthology.orchestra.run.vm06:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2026-03-24T02:13:18.428 INFO:tasks.ceph:Reading keys from all nodes... 2026-03-24T02:13:18.428 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:18.428 DEBUG:teuthology.orchestra.run.vm06:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2026-03-24T02:13:18.479 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:18.479 DEBUG:teuthology.orchestra.run.vm06:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2026-03-24T02:13:18.528 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:18.528 DEBUG:teuthology.orchestra.run.vm06:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2026-03-24T02:13:18.580 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:18.580 DEBUG:teuthology.orchestra.run.vm06:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2026-03-24T02:13:18.633 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:18.633 DEBUG:teuthology.orchestra.run.vm06:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2026-03-24T02:13:18.678 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:18.679 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2026-03-24T02:13:18.682 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:18.682 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2026-03-24T02:13:18.727 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:18.727 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2026-03-24T02:13:18.770 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:18.770 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2026-03-24T02:13:18.814 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:18.814 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2026-03-24T02:13:18.858 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:18.858 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2026-03-24T02:13:18.902 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:18.902 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2026-03-24T02:13:18.946 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:18.946 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2026-03-24T02:13:18.990 INFO:tasks.ceph:Adding keys to all mons... 2026-03-24T02:13:18.990 DEBUG:teuthology.orchestra.run.vm06:> sudo tee -a /etc/ceph/cluster1.keyring 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout:[mgr.x] 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout: key = AQA688FpaYlBDBAAMNFuPpA9BmtqJsU+XTE1tg== 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout:[osd.0] 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout: key = AQA788FpoqJqHBAA4UNzgwKR5TEtmF14a/ncGQ== 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout:[osd.1] 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout: key = AQA888Fpzec4HBAA/CoXcj8ULWVAwAY/OoRuKQ== 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout:[osd.2] 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout: key = AQA988FpcqyoHBAAw6zQAsCjCmAa9xsTvCVN3w== 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout:[client.0] 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout: key = AQA688FpszHgDxAAoZ6bKDo83A7qvFcMAKjTww== 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout:[client.mirror] 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout: key = AQA688FpKYaAERAAOxraWo2temFqR9WiveBVRA== 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout:[client.mirror.0] 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout: key = AQA688FpzpeaFRAArS5MDmAi5cYJl2sR+6MhJg== 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout:[client.mirror.1] 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout: key = AQA688Fpekm7GRAAMhIopHgVDYLYvlQ3uSdeYA== 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout:[client.mirror.2] 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout: key = AQA688FpXR7FHRAA4grqZRb8dCATG6ua++0Xwg== 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout:[client.mirror.3] 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout: key = AQA688FpbSzMIRAAvTDMMJH70T5hd4qEd4B4gw== 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout:[client.mirror.4] 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout: key = AQA688Fp0cbdJRAAOwDzVUTRy2pKt9CR85+EGw== 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout:[client.mirror.5] 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout: key = AQA688FpwnviKRAA8ZG2EaL57E5vQ9pvFjmo0w== 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout:[client.mirror.6] 2026-03-24T02:13:18.999 INFO:teuthology.orchestra.run.vm06.stdout: key = AQA688Fp/F/9MBAAHhd/IKNZJq5fIqrTWlESZg== 2026-03-24T02:13:19.000 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-03-24T02:13:19.067 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-24T02:13:19.132 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-24T02:13:19.199 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-24T02:13:19.260 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:19.333 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:19.407 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:19.476 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:19.549 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:19.616 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:19.684 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:19.756 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:19.824 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:19.892 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-03-24T02:13:19.892 DEBUG:teuthology.orchestra.run.vm06:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2026-03-24T02:13:19.944 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2026-03-24T02:13:20.049 DEBUG:teuthology.orchestra.run.vm06:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2026-03-24T02:13:20.058 DEBUG:teuthology.orchestra.run.vm06:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2026-03-24T02:13:20.106 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2026-03-24T02:13:20.107 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2026-03-24T02:13:20.107 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2026-03-24T02:13:20.152 INFO:tasks.ceph.cluster1.mon.a:Started 2026-03-24T02:13:20.152 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2026-03-24T02:13:20.152 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2026-03-24T02:13:20.152 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2026-03-24T02:13:20.153 INFO:tasks.ceph.cluster1.mgr.x:Started 2026-03-24T02:13:20.153 DEBUG:tasks.ceph:set 0 configs 2026-03-24T02:13:20.153 DEBUG:teuthology.orchestra.run.vm06:> sudo ceph --cluster cluster1 config dump 2026-03-24T02:13:20.340 INFO:teuthology.orchestra.run.vm06.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-03-24T02:13:20.354 INFO:tasks.ceph:Setting crush tunables to default 2026-03-24T02:13:20.354 DEBUG:teuthology.orchestra.run.vm06:> sudo ceph --cluster cluster1 osd crush tunables default 2026-03-24T02:13:20.466 INFO:teuthology.orchestra.run.vm06.stderr:adjusted tunables profile to default 2026-03-24T02:13:20.478 INFO:tasks.ceph:check_enable_crimson: False 2026-03-24T02:13:20.478 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2026-03-24T02:13:20.478 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:20.478 DEBUG:teuthology.orchestra.run.vm06:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2026-03-24T02:13:20.488 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:20.488 DEBUG:teuthology.orchestra.run.vm06:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2026-03-24T02:13:20.540 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:20.540 DEBUG:teuthology.orchestra.run.vm06:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2026-03-24T02:13:20.592 DEBUG:teuthology.orchestra.run.vm06:> sudo ceph --cluster cluster1 osd new 7df1aacf-a228-4799-af31-c87d813181b3 0 2026-03-24T02:13:20.909 INFO:teuthology.orchestra.run.vm06.stdout:0 2026-03-24T02:13:20.925 DEBUG:teuthology.orchestra.run.vm06:> sudo ceph --cluster cluster1 osd new ac96531a-b5de-4d84-959b-01dcd619f954 1 2026-03-24T02:13:21.044 INFO:teuthology.orchestra.run.vm06.stdout:1 2026-03-24T02:13:21.056 DEBUG:teuthology.orchestra.run.vm06:> sudo ceph --cluster cluster1 osd new 57b8ad53-f9a4-4e46-be96-93ad890cbba4 2 2026-03-24T02:13:21.180 INFO:teuthology.orchestra.run.vm06.stdout:2 2026-03-24T02:13:21.198 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2026-03-24T02:13:21.198 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2026-03-24T02:13:21.199 INFO:tasks.ceph.cluster1.osd.0:Started 2026-03-24T02:13:21.199 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2026-03-24T02:13:21.200 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2026-03-24T02:13:21.200 INFO:tasks.ceph.cluster1.osd.1:Started 2026-03-24T02:13:21.200 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2026-03-24T02:13:21.200 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2026-03-24T02:13:21.201 INFO:tasks.ceph.cluster1.osd.2:Started 2026-03-24T02:13:21.202 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-03-24T02:13:21.332 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:21.332 INFO:teuthology.orchestra.run.vm06.stdout:{"epoch":5,"fsid":"91306d06-3c56-4223-8b51-5365acf96311","created":"2026-03-24T02:13:20.206909+0000","modified":"2026-03-24T02:13:21.167197+0000","last_up_change":"0.000000","last_in_change":"2026-03-24T02:13:21.167197+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"7df1aacf-a228-4799-af31-c87d813181b3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"ac96531a-b5de-4d84-959b-01dcd619f954","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"57b8ad53-f9a4-4e46-be96-93ad890cbba4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-24T02:13:21.347 INFO:tasks.ceph.ceph_manager.cluster1:[] 2026-03-24T02:13:21.348 INFO:tasks.ceph:Waiting for OSDs to come up 2026-03-24T02:13:21.532 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:13:21.530+0000 7fb287026a40 -1 Falling back to public interface 2026-03-24T02:13:21.548 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:13:21.546+0000 7fc16783da40 -1 Falling back to public interface 2026-03-24T02:13:21.548 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:13:21.546+0000 7f54a4641a40 -1 Falling back to public interface 2026-03-24T02:13:21.615 INFO:tasks.ceph.cluster1.mgr.x.vm06.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-03-24T02:13:21.630 INFO:tasks.ceph.cluster1.mgr.x.vm06.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-03-24T02:13:21.630 INFO:tasks.ceph.cluster1.mgr.x.vm06.stderr: from numpy import show_config as show_numpy_config 2026-03-24T02:13:21.649 DEBUG:teuthology.orchestra.run.vm06:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-03-24T02:13:21.772 INFO:teuthology.misc.health.vm06.stdout: 2026-03-24T02:13:21.773 INFO:teuthology.misc.health.vm06.stdout:{"epoch":5,"fsid":"91306d06-3c56-4223-8b51-5365acf96311","created":"2026-03-24T02:13:20.206909+0000","modified":"2026-03-24T02:13:21.167197+0000","last_up_change":"0.000000","last_in_change":"2026-03-24T02:13:21.167197+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"7df1aacf-a228-4799-af31-c87d813181b3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"ac96531a-b5de-4d84-959b-01dcd619f954","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"57b8ad53-f9a4-4e46-be96-93ad890cbba4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-24T02:13:21.792 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-03-24T02:13:22.151 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:13:22.146+0000 7fb287026a40 -1 osd.1 0 log_to_monitors true 2026-03-24T02:13:22.177 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:13:22.174+0000 7fc16783da40 -1 osd.0 0 log_to_monitors true 2026-03-24T02:13:22.215 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:13:22.210+0000 7f54a4641a40 -1 osd.2 0 log_to_monitors true 2026-03-24T02:13:22.256 INFO:tasks.ceph.cluster1.mgr.x.vm06.stderr:Failed to import NVMeoFClient and related components: cannot import name 'NVMeoFClient' from 'dashboard.services.nvmeof_client' (/usr/share/ceph/mgr/dashboard/services/nvmeof_client.py) 2026-03-24T02:13:23.320 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:13:23.318+0000 7fc1637e6640 -1 osd.0 0 waiting for initial osdmap 2026-03-24T02:13:23.320 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:13:23.318+0000 7fb282fcf640 -1 osd.1 0 waiting for initial osdmap 2026-03-24T02:13:23.320 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:13:23.318+0000 7f54a05ea640 -1 osd.2 0 waiting for initial osdmap 2026-03-24T02:13:23.322 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:13:23.318+0000 7fc15e5f4640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-24T02:13:23.323 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:13:23.318+0000 7f549b3f8640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-24T02:13:23.323 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:13:23.318+0000 7fb27dddd640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-24T02:13:23.762 INFO:tasks.ceph.cluster1.mgr.x.vm06.stderr:2026-03-24T02:13:23.758+0000 7f85fc55d640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-03-24T02:13:28.094 DEBUG:teuthology.orchestra.run.vm06:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-03-24T02:13:28.251 INFO:teuthology.misc.health.vm06.stdout: 2026-03-24T02:13:28.251 INFO:teuthology.misc.health.vm06.stdout:{"epoch":11,"fsid":"91306d06-3c56-4223-8b51-5365acf96311","created":"2026-03-24T02:13:20.206909+0000","modified":"2026-03-24T02:13:27.332018+0000","last_up_change":"2026-03-24T02:13:24.307479+0000","last_in_change":"2026-03-24T02:13:21.167197+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-24T02:13:24.769197+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"7df1aacf-a228-4799-af31-c87d813181b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6809","nonce":178400628},{"type":"v1","addr":"192.168.123.106:6811","nonce":178400628}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6813","nonce":178400628},{"type":"v1","addr":"192.168.123.106:6815","nonce":178400628}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6821","nonce":178400628},{"type":"v1","addr":"192.168.123.106:6823","nonce":178400628}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6817","nonce":178400628},{"type":"v1","addr":"192.168.123.106:6819","nonce":178400628}]},"public_addr":"192.168.123.106:6811/178400628","cluster_addr":"192.168.123.106:6815/178400628","heartbeat_back_addr":"192.168.123.106:6823/178400628","heartbeat_front_addr":"192.168.123.106:6819/178400628","state":["exists","up"]},{"osd":1,"uuid":"ac96531a-b5de-4d84-959b-01dcd619f954","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6800","nonce":2062373619},{"type":"v1","addr":"192.168.123.106:6801","nonce":2062373619}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6802","nonce":2062373619},{"type":"v1","addr":"192.168.123.106:6803","nonce":2062373619}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6806","nonce":2062373619},{"type":"v1","addr":"192.168.123.106:6807","nonce":2062373619}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6804","nonce":2062373619},{"type":"v1","addr":"192.168.123.106:6805","nonce":2062373619}]},"public_addr":"192.168.123.106:6801/2062373619","cluster_addr":"192.168.123.106:6803/2062373619","heartbeat_back_addr":"192.168.123.106:6807/2062373619","heartbeat_front_addr":"192.168.123.106:6805/2062373619","state":["exists","up"]},{"osd":2,"uuid":"57b8ad53-f9a4-4e46-be96-93ad890cbba4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6808","nonce":1501752565},{"type":"v1","addr":"192.168.123.106:6810","nonce":1501752565}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6812","nonce":1501752565},{"type":"v1","addr":"192.168.123.106:6814","nonce":1501752565}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6820","nonce":1501752565},{"type":"v1","addr":"192.168.123.106:6822","nonce":1501752565}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6816","nonce":1501752565},{"type":"v1","addr":"192.168.123.106:6818","nonce":1501752565}]},"public_addr":"192.168.123.106:6810/1501752565","cluster_addr":"192.168.123.106:6814/1501752565","heartbeat_back_addr":"192.168.123.106:6822/1501752565","heartbeat_front_addr":"192.168.123.106:6818/1501752565","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-24T02:13:28.264 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-03-24T02:13:28.264 INFO:tasks.ceph:Creating RBD pool 2026-03-24T02:13:28.264 DEBUG:teuthology.orchestra.run.vm06:> sudo ceph --cluster cluster1 osd pool create rbd 8 2026-03-24T02:13:29.341 INFO:teuthology.orchestra.run.vm06.stderr:pool 'rbd' created 2026-03-24T02:13:29.361 DEBUG:teuthology.orchestra.run.vm06:> rbd --cluster cluster1 pool init rbd 2026-03-24T02:13:32.414 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2026-03-24T02:13:32.414 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 config log 1 --format=json 2026-03-24T02:13:32.414 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-03-24T02:13:32.579 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:32.592 INFO:teuthology.orchestra.run.vm06.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-03-24T02:13:32.592 INFO:tasks.ceph_manager:config epoch is 1 2026-03-24T02:13:32.592 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2026-03-24T02:13:32.592 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2026-03-24T02:13:32.592 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2026-03-24T02:13:32.783 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:32.797 INFO:teuthology.orchestra.run.vm06.stdout:{"epoch":5,"flags":0,"active_gid":4101,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6824","nonce":2423370535},{"type":"v1","addr":"192.168.123.106:6825","nonce":2423370535}]},"active_addr":"192.168.123.106:6825/2423370535","active_change":"2026-03-24T02:13:22.733609+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self certificates generated and signed by cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.5","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"how frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"192.168.123.106:0","nonce":1987944628}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"192.168.123.106:0","nonce":25172831}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"192.168.123.106:0","nonce":3086005403}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"192.168.123.106:0","nonce":833423356}]}]} 2026-03-24T02:13:32.798 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2026-03-24T02:13:32.798 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2026-03-24T02:13:32.798 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-03-24T02:13:32.978 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:32.978 INFO:teuthology.orchestra.run.vm06.stdout:{"epoch":15,"fsid":"91306d06-3c56-4223-8b51-5365acf96311","created":"2026-03-24T02:13:20.206909+0000","modified":"2026-03-24T02:13:32.400386+0000","last_up_change":"2026-03-24T02:13:24.307479+0000","last_in_change":"2026-03-24T02:13:21.167197+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-24T02:13:24.769197+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-03-24T02:13:28.442828+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"7df1aacf-a228-4799-af31-c87d813181b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6809","nonce":178400628},{"type":"v1","addr":"192.168.123.106:6811","nonce":178400628}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6813","nonce":178400628},{"type":"v1","addr":"192.168.123.106:6815","nonce":178400628}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6821","nonce":178400628},{"type":"v1","addr":"192.168.123.106:6823","nonce":178400628}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6817","nonce":178400628},{"type":"v1","addr":"192.168.123.106:6819","nonce":178400628}]},"public_addr":"192.168.123.106:6811/178400628","cluster_addr":"192.168.123.106:6815/178400628","heartbeat_back_addr":"192.168.123.106:6823/178400628","heartbeat_front_addr":"192.168.123.106:6819/178400628","state":["exists","up"]},{"osd":1,"uuid":"ac96531a-b5de-4d84-959b-01dcd619f954","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6800","nonce":2062373619},{"type":"v1","addr":"192.168.123.106:6801","nonce":2062373619}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6802","nonce":2062373619},{"type":"v1","addr":"192.168.123.106:6803","nonce":2062373619}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6806","nonce":2062373619},{"type":"v1","addr":"192.168.123.106:6807","nonce":2062373619}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6804","nonce":2062373619},{"type":"v1","addr":"192.168.123.106:6805","nonce":2062373619}]},"public_addr":"192.168.123.106:6801/2062373619","cluster_addr":"192.168.123.106:6803/2062373619","heartbeat_back_addr":"192.168.123.106:6807/2062373619","heartbeat_front_addr":"192.168.123.106:6805/2062373619","state":["exists","up"]},{"osd":2,"uuid":"57b8ad53-f9a4-4e46-be96-93ad890cbba4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6808","nonce":1501752565},{"type":"v1","addr":"192.168.123.106:6810","nonce":1501752565}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6812","nonce":1501752565},{"type":"v1","addr":"192.168.123.106:6814","nonce":1501752565}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6820","nonce":1501752565},{"type":"v1","addr":"192.168.123.106:6822","nonce":1501752565}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6816","nonce":1501752565},{"type":"v1","addr":"192.168.123.106:6818","nonce":1501752565}]},"public_addr":"192.168.123.106:6810/1501752565","cluster_addr":"192.168.123.106:6814/1501752565","heartbeat_back_addr":"192.168.123.106:6822/1501752565","heartbeat_front_addr":"192.168.123.106:6818/1501752565","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-24T02:13:32.994 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2026-03-24T02:13:32.994 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-03-24T02:13:33.172 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:33.172 INFO:teuthology.orchestra.run.vm06.stdout:{"epoch":15,"fsid":"91306d06-3c56-4223-8b51-5365acf96311","created":"2026-03-24T02:13:20.206909+0000","modified":"2026-03-24T02:13:32.400386+0000","last_up_change":"2026-03-24T02:13:24.307479+0000","last_in_change":"2026-03-24T02:13:21.167197+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-24T02:13:24.769197+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-03-24T02:13:28.442828+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"7df1aacf-a228-4799-af31-c87d813181b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6809","nonce":178400628},{"type":"v1","addr":"192.168.123.106:6811","nonce":178400628}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6813","nonce":178400628},{"type":"v1","addr":"192.168.123.106:6815","nonce":178400628}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6821","nonce":178400628},{"type":"v1","addr":"192.168.123.106:6823","nonce":178400628}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6817","nonce":178400628},{"type":"v1","addr":"192.168.123.106:6819","nonce":178400628}]},"public_addr":"192.168.123.106:6811/178400628","cluster_addr":"192.168.123.106:6815/178400628","heartbeat_back_addr":"192.168.123.106:6823/178400628","heartbeat_front_addr":"192.168.123.106:6819/178400628","state":["exists","up"]},{"osd":1,"uuid":"ac96531a-b5de-4d84-959b-01dcd619f954","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6800","nonce":2062373619},{"type":"v1","addr":"192.168.123.106:6801","nonce":2062373619}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6802","nonce":2062373619},{"type":"v1","addr":"192.168.123.106:6803","nonce":2062373619}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6806","nonce":2062373619},{"type":"v1","addr":"192.168.123.106:6807","nonce":2062373619}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6804","nonce":2062373619},{"type":"v1","addr":"192.168.123.106:6805","nonce":2062373619}]},"public_addr":"192.168.123.106:6801/2062373619","cluster_addr":"192.168.123.106:6803/2062373619","heartbeat_back_addr":"192.168.123.106:6807/2062373619","heartbeat_front_addr":"192.168.123.106:6805/2062373619","state":["exists","up"]},{"osd":2,"uuid":"57b8ad53-f9a4-4e46-be96-93ad890cbba4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6808","nonce":1501752565},{"type":"v1","addr":"192.168.123.106:6810","nonce":1501752565}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6812","nonce":1501752565},{"type":"v1","addr":"192.168.123.106:6814","nonce":1501752565}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6820","nonce":1501752565},{"type":"v1","addr":"192.168.123.106:6822","nonce":1501752565}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6816","nonce":1501752565},{"type":"v1","addr":"192.168.123.106:6818","nonce":1501752565}]},"public_addr":"192.168.123.106:6810/1501752565","cluster_addr":"192.168.123.106:6814/1501752565","heartbeat_back_addr":"192.168.123.106:6822/1501752565","heartbeat_front_addr":"192.168.123.106:6818/1501752565","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-24T02:13:33.186 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2026-03-24T02:13:33.186 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2026-03-24T02:13:33.186 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2026-03-24T02:13:33.294 INFO:teuthology.orchestra.run.vm06.stdout:34359738371 2026-03-24T02:13:33.294 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-03-24T02:13:33.295 INFO:teuthology.orchestra.run.vm06.stdout:34359738371 2026-03-24T02:13:33.295 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-03-24T02:13:33.299 INFO:teuthology.orchestra.run.vm06.stdout:34359738371 2026-03-24T02:13:33.299 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-03-24T02:13:33.489 INFO:teuthology.orchestra.run.vm06.stdout:34359738370 2026-03-24T02:13:33.505 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.0 2026-03-24T02:13:33.519 INFO:teuthology.orchestra.run.vm06.stdout:34359738370 2026-03-24T02:13:33.523 INFO:teuthology.orchestra.run.vm06.stdout:34359738370 2026-03-24T02:13:33.535 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.1 2026-03-24T02:13:33.538 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.2 2026-03-24T02:13:34.505 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-03-24T02:13:34.536 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-03-24T02:13:34.539 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-03-24T02:13:34.685 INFO:teuthology.orchestra.run.vm06.stdout:34359738370 2026-03-24T02:13:34.699 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.0 2026-03-24T02:13:34.722 INFO:teuthology.orchestra.run.vm06.stdout:34359738370 2026-03-24T02:13:34.728 INFO:teuthology.orchestra.run.vm06.stdout:34359738370 2026-03-24T02:13:34.736 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.1 2026-03-24T02:13:34.746 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.2 2026-03-24T02:13:35.699 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-03-24T02:13:35.747 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-03-24T02:13:35.747 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-03-24T02:13:35.875 INFO:teuthology.orchestra.run.vm06.stdout:34359738371 2026-03-24T02:13:35.889 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.0 2026-03-24T02:13:35.889 DEBUG:teuthology.parallel:result is None 2026-03-24T02:13:35.929 INFO:teuthology.orchestra.run.vm06.stdout:34359738371 2026-03-24T02:13:35.933 INFO:teuthology.orchestra.run.vm06.stdout:34359738371 2026-03-24T02:13:35.944 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.2 2026-03-24T02:13:35.944 DEBUG:teuthology.parallel:result is None 2026-03-24T02:13:35.948 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.1 2026-03-24T02:13:35.948 DEBUG:teuthology.parallel:result is None 2026-03-24T02:13:35.948 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-03-24T02:13:35.948 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-03-24T02:13:36.176 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:36.176 INFO:teuthology.orchestra.run.vm06.stderr:dumped all 2026-03-24T02:13:36.192 INFO:teuthology.orchestra.run.vm06.stdout:{"pg_ready":true,"pg_map":{"version":17,"stamp":"2026-03-24T02:13:34.758671+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":459299,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":59,"num_write_kb":586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35,"ondisk_log_size":35,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":17,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81840,"kb_used_data":1296,"kb_used_omap":25,"kb_used_meta":80422,"kb_avail":283033680,"statfs":{"total":289910292480,"available":289826488320,"internally_reserved":0,"allocated":1327104,"data_stored":1032955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25661,"internal_metadata":82353091},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.415189"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.410422+0000","last_change":"2026-03-24T02:13:32.410665+0000","last_active":"2026-03-24T02:13:32.410422+0000","last_peered":"2026-03-24T02:13:32.410422+0000","last_clean":"2026-03-24T02:13:32.410422+0000","last_became_active":"2026-03-24T02:13:30.350133+0000","last_became_peered":"2026-03-24T02:13:30.350133+0000","last_unstale":"2026-03-24T02:13:32.410422+0000","last_undegraded":"2026-03-24T02:13:32.410422+0000","last_fullsized":"2026-03-24T02:13:32.410422+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:51:40.295988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035299000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.410468+0000","last_change":"2026-03-24T02:13:32.410938+0000","last_active":"2026-03-24T02:13:32.410468+0000","last_peered":"2026-03-24T02:13:32.410468+0000","last_clean":"2026-03-24T02:13:32.410468+0000","last_became_active":"2026-03-24T02:13:30.349481+0000","last_became_peered":"2026-03-24T02:13:30.349481+0000","last_unstale":"2026-03-24T02:13:32.410468+0000","last_undegraded":"2026-03-24T02:13:32.410468+0000","last_fullsized":"2026-03-24T02:13:32.410468+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:35:42.157836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055518799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.410691+0000","last_change":"2026-03-24T02:13:32.410738+0000","last_active":"2026-03-24T02:13:32.410691+0000","last_peered":"2026-03-24T02:13:32.410691+0000","last_clean":"2026-03-24T02:13:32.410691+0000","last_became_active":"2026-03-24T02:13:30.348547+0000","last_became_peered":"2026-03-24T02:13:30.348547+0000","last_unstale":"2026-03-24T02:13:32.410691+0000","last_undegraded":"2026-03-24T02:13:32.410691+0000","last_fullsized":"2026-03-24T02:13:32.410691+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:52:23.166796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033254200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.410963+0000","last_change":"2026-03-24T02:13:32.411084+0000","last_active":"2026-03-24T02:13:32.410963+0000","last_peered":"2026-03-24T02:13:32.410963+0000","last_clean":"2026-03-24T02:13:32.410963+0000","last_became_active":"2026-03-24T02:13:30.349564+0000","last_became_peered":"2026-03-24T02:13:30.349564+0000","last_unstale":"2026-03-24T02:13:32.410963+0000","last_undegraded":"2026-03-24T02:13:32.410963+0000","last_fullsized":"2026-03-24T02:13:32.410963+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:04:43.950912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059685599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":22,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.409322+0000","last_change":"2026-03-24T02:13:32.409322+0000","last_active":"2026-03-24T02:13:32.409322+0000","last_peered":"2026-03-24T02:13:32.409322+0000","last_clean":"2026-03-24T02:13:32.409322+0000","last_became_active":"2026-03-24T02:13:30.350751+0000","last_became_peered":"2026-03-24T02:13:30.350751+0000","last_unstale":"2026-03-24T02:13:32.409322+0000","last_undegraded":"2026-03-24T02:13:32.409322+0000","last_fullsized":"2026-03-24T02:13:32.409322+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:58:18.488299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082885000000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.406527+0000","last_change":"2026-03-24T02:13:32.406625+0000","last_active":"2026-03-24T02:13:32.406527+0000","last_peered":"2026-03-24T02:13:32.406527+0000","last_clean":"2026-03-24T02:13:32.406527+0000","last_became_active":"2026-03-24T02:13:30.348773+0000","last_became_peered":"2026-03-24T02:13:30.348773+0000","last_unstale":"2026-03-24T02:13:32.406527+0000","last_undegraded":"2026-03-24T02:13:32.406527+0000","last_fullsized":"2026-03-24T02:13:32.406527+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T08:00:33.758016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00018693,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.406635+0000","last_change":"2026-03-24T02:13:32.406690+0000","last_active":"2026-03-24T02:13:32.406635+0000","last_peered":"2026-03-24T02:13:32.406635+0000","last_clean":"2026-03-24T02:13:32.406635+0000","last_became_active":"2026-03-24T02:13:30.348538+0000","last_became_peered":"2026-03-24T02:13:30.348538+0000","last_unstale":"2026-03-24T02:13:32.406635+0000","last_undegraded":"2026-03-24T02:13:32.406635+0000","last_fullsized":"2026-03-24T02:13:32.406635+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T13:03:07.820972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023985799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.411069+0000","last_change":"2026-03-24T02:13:32.411138+0000","last_active":"2026-03-24T02:13:32.411069+0000","last_peered":"2026-03-24T02:13:32.411069+0000","last_clean":"2026-03-24T02:13:32.411069+0000","last_became_active":"2026-03-24T02:13:30.348335+0000","last_became_peered":"2026-03-24T02:13:30.348335+0000","last_unstale":"2026-03-24T02:13:32.411069+0000","last_undegraded":"2026-03-24T02:13:32.411069+0000","last_fullsized":"2026-03-24T02:13:32.411069+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:37:39.995117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058244000000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'32","reported_seq":65,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.410517+0000","last_change":"2026-03-24T02:13:27.310722+0000","last_active":"2026-03-24T02:13:32.410517+0000","last_peered":"2026-03-24T02:13:32.410517+0000","last_clean":"2026-03-24T02:13:32.410517+0000","last_became_active":"2026-03-24T02:13:27.310550+0000","last_became_peered":"2026-03-24T02:13:27.310550+0000","last_unstale":"2026-03-24T02:13:32.410517+0000","last_undegraded":"2026-03-24T02:13:32.410517+0000","last_fullsized":"2026-03-24T02:13:32.410517+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:25.316656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:25.316656+0000","last_clean_scrub_stamp":"2026-03-24T02:13:25.316656+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:46:01.042093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":925696,"data_stored":918560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32,"ondisk_log_size":32,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738371,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":26976,"kb_used_data":128,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":94344864,"statfs":{"total":96636764160,"available":96609140736,"internally_reserved":0,"allocated":131072,"data_stored":38119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8121,"internal_metadata":27451463},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738371,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27432,"kb_used_data":584,"kb_used_omap":8,"kb_used_meta":26807,"kb_avail":94344408,"statfs":{"total":96636764160,"available":96608673792,"internally_reserved":0,"allocated":598016,"data_stored":497418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8771,"internal_metadata":27450813},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27432,"kb_used_data":584,"kb_used_omap":8,"kb_used_meta":26807,"kb_avail":94344408,"statfs":{"total":96636764160,"available":96608673792,"internally_reserved":0,"allocated":598016,"data_stored":497418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8769,"internal_metadata":27450815},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-24T02:13:36.192 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-03-24T02:13:36.371 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:36.371 INFO:teuthology.orchestra.run.vm06.stderr:dumped all 2026-03-24T02:13:36.387 INFO:teuthology.orchestra.run.vm06.stdout:{"pg_ready":true,"pg_map":{"version":17,"stamp":"2026-03-24T02:13:34.758671+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":459299,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":59,"num_write_kb":586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35,"ondisk_log_size":35,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":17,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81840,"kb_used_data":1296,"kb_used_omap":25,"kb_used_meta":80422,"kb_avail":283033680,"statfs":{"total":289910292480,"available":289826488320,"internally_reserved":0,"allocated":1327104,"data_stored":1032955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25661,"internal_metadata":82353091},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.415189"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.410422+0000","last_change":"2026-03-24T02:13:32.410665+0000","last_active":"2026-03-24T02:13:32.410422+0000","last_peered":"2026-03-24T02:13:32.410422+0000","last_clean":"2026-03-24T02:13:32.410422+0000","last_became_active":"2026-03-24T02:13:30.350133+0000","last_became_peered":"2026-03-24T02:13:30.350133+0000","last_unstale":"2026-03-24T02:13:32.410422+0000","last_undegraded":"2026-03-24T02:13:32.410422+0000","last_fullsized":"2026-03-24T02:13:32.410422+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:51:40.295988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035299000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.410468+0000","last_change":"2026-03-24T02:13:32.410938+0000","last_active":"2026-03-24T02:13:32.410468+0000","last_peered":"2026-03-24T02:13:32.410468+0000","last_clean":"2026-03-24T02:13:32.410468+0000","last_became_active":"2026-03-24T02:13:30.349481+0000","last_became_peered":"2026-03-24T02:13:30.349481+0000","last_unstale":"2026-03-24T02:13:32.410468+0000","last_undegraded":"2026-03-24T02:13:32.410468+0000","last_fullsized":"2026-03-24T02:13:32.410468+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:35:42.157836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055518799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.410691+0000","last_change":"2026-03-24T02:13:32.410738+0000","last_active":"2026-03-24T02:13:32.410691+0000","last_peered":"2026-03-24T02:13:32.410691+0000","last_clean":"2026-03-24T02:13:32.410691+0000","last_became_active":"2026-03-24T02:13:30.348547+0000","last_became_peered":"2026-03-24T02:13:30.348547+0000","last_unstale":"2026-03-24T02:13:32.410691+0000","last_undegraded":"2026-03-24T02:13:32.410691+0000","last_fullsized":"2026-03-24T02:13:32.410691+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:52:23.166796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033254200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.410963+0000","last_change":"2026-03-24T02:13:32.411084+0000","last_active":"2026-03-24T02:13:32.410963+0000","last_peered":"2026-03-24T02:13:32.410963+0000","last_clean":"2026-03-24T02:13:32.410963+0000","last_became_active":"2026-03-24T02:13:30.349564+0000","last_became_peered":"2026-03-24T02:13:30.349564+0000","last_unstale":"2026-03-24T02:13:32.410963+0000","last_undegraded":"2026-03-24T02:13:32.410963+0000","last_fullsized":"2026-03-24T02:13:32.410963+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:04:43.950912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059685599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":22,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.409322+0000","last_change":"2026-03-24T02:13:32.409322+0000","last_active":"2026-03-24T02:13:32.409322+0000","last_peered":"2026-03-24T02:13:32.409322+0000","last_clean":"2026-03-24T02:13:32.409322+0000","last_became_active":"2026-03-24T02:13:30.350751+0000","last_became_peered":"2026-03-24T02:13:30.350751+0000","last_unstale":"2026-03-24T02:13:32.409322+0000","last_undegraded":"2026-03-24T02:13:32.409322+0000","last_fullsized":"2026-03-24T02:13:32.409322+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:58:18.488299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082885000000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.406527+0000","last_change":"2026-03-24T02:13:32.406625+0000","last_active":"2026-03-24T02:13:32.406527+0000","last_peered":"2026-03-24T02:13:32.406527+0000","last_clean":"2026-03-24T02:13:32.406527+0000","last_became_active":"2026-03-24T02:13:30.348773+0000","last_became_peered":"2026-03-24T02:13:30.348773+0000","last_unstale":"2026-03-24T02:13:32.406527+0000","last_undegraded":"2026-03-24T02:13:32.406527+0000","last_fullsized":"2026-03-24T02:13:32.406527+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T08:00:33.758016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00018693,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.406635+0000","last_change":"2026-03-24T02:13:32.406690+0000","last_active":"2026-03-24T02:13:32.406635+0000","last_peered":"2026-03-24T02:13:32.406635+0000","last_clean":"2026-03-24T02:13:32.406635+0000","last_became_active":"2026-03-24T02:13:30.348538+0000","last_became_peered":"2026-03-24T02:13:30.348538+0000","last_unstale":"2026-03-24T02:13:32.406635+0000","last_undegraded":"2026-03-24T02:13:32.406635+0000","last_fullsized":"2026-03-24T02:13:32.406635+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T13:03:07.820972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023985799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.411069+0000","last_change":"2026-03-24T02:13:32.411138+0000","last_active":"2026-03-24T02:13:32.411069+0000","last_peered":"2026-03-24T02:13:32.411069+0000","last_clean":"2026-03-24T02:13:32.411069+0000","last_became_active":"2026-03-24T02:13:30.348335+0000","last_became_peered":"2026-03-24T02:13:30.348335+0000","last_unstale":"2026-03-24T02:13:32.411069+0000","last_undegraded":"2026-03-24T02:13:32.411069+0000","last_fullsized":"2026-03-24T02:13:32.411069+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:37:39.995117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058244000000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'32","reported_seq":65,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-24T02:13:32.410517+0000","last_change":"2026-03-24T02:13:27.310722+0000","last_active":"2026-03-24T02:13:32.410517+0000","last_peered":"2026-03-24T02:13:32.410517+0000","last_clean":"2026-03-24T02:13:32.410517+0000","last_became_active":"2026-03-24T02:13:27.310550+0000","last_became_peered":"2026-03-24T02:13:27.310550+0000","last_unstale":"2026-03-24T02:13:32.410517+0000","last_undegraded":"2026-03-24T02:13:32.410517+0000","last_fullsized":"2026-03-24T02:13:32.410517+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:25.316656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:25.316656+0000","last_clean_scrub_stamp":"2026-03-24T02:13:25.316656+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:46:01.042093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":925696,"data_stored":918560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32,"ondisk_log_size":32,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738371,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":26976,"kb_used_data":128,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":94344864,"statfs":{"total":96636764160,"available":96609140736,"internally_reserved":0,"allocated":131072,"data_stored":38119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8121,"internal_metadata":27451463},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738371,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27432,"kb_used_data":584,"kb_used_omap":8,"kb_used_meta":26807,"kb_avail":94344408,"statfs":{"total":96636764160,"available":96608673792,"internally_reserved":0,"allocated":598016,"data_stored":497418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8771,"internal_metadata":27450813},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27432,"kb_used_data":584,"kb_used_omap":8,"kb_used_meta":26807,"kb_avail":94344408,"statfs":{"total":96636764160,"available":96608673792,"internally_reserved":0,"allocated":598016,"data_stored":497418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8769,"internal_metadata":27450815},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-24T02:13:36.387 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-03-24T02:13:36.387 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2026-03-24T02:13:36.387 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2026-03-24T02:13:36.387 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2026-03-24T02:13:36.576 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:36.577 INFO:teuthology.orchestra.run.vm06.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-03-24T02:13:36.591 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2026-03-24T02:13:36.591 INFO:teuthology.run_tasks:Running task ceph... 2026-03-24T02:13:36.593 INFO:tasks.ceph:Creating ceph cluster cluster2... 2026-03-24T02:13:36.593 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-03-24T02:13:36.593 INFO:tasks.ceph:ctx.config {'archive_path': '/archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3539', 'branch': 'tentacle', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-journal-stress-workunit}', 'email': None, 'first_in_suite': False, 'flavor': 'default', 'job_id': '3539', 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'vps', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps', 'no_nested_subset': False, 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': 'tentacle'}, 'ansible.cephlab': {'branch': 'main', 'repo': 'https://github.com/kshtsk/ceph-cm-ansible.git', 'skip_tags': 'nagios,monitoring-scripts,hostname,pubkeys,zap,sudoers,kerberos,ntp-client,resolvconf,cpan,nfs', 'vars': {'logical_volumes': {'lv_1': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_2': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_3': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_4': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}}, 'timezone': 'UTC', 'volume_groups': {'vg_nvme': {'pvs': '/dev/vdb,/dev/vdc,/dev/vdd,/dev/vde'}}}}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'cephadm': {'cephadm_binary_url': 'https://download.ceph.com/rpm-20.2.0/el9/noarch/cephadm'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}, 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'tt-tentacle', 'sha1': '0392f78529848ec72469e8e431875cb98d3a5fb4'}}, 'owner': 'kyr', 'priority': 1000, 'repo': 'https://github.com/ceph/ceph.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 3051, 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'sleep_before_teardown': 0, 'subset': '1/128', 'suite': 'rbd', 'suite_branch': 'tt-tentacle', 'suite_path': '/home/teuthos/src/github.com_kshtsk_ceph_0392f78529848ec72469e8e431875cb98d3a5fb4/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/kshtsk/ceph.git', 'suite_sha1': '0392f78529848ec72469e8e431875cb98d3a5fb4', 'targets': {'vm06.local': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFMdzVQ4qWHGmJfQDRxmfOu8MjX2Enb0f1yGF9w/uaSFB9MTnpB4aag7G6CcWBNqpWlI9EJwSapJNyvZr8HZKHc=', 'vm07.local': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK5C8C/vVG/hbJJiioODjMgoQKxU7Psh6pK8go4Z7ooenLW0HVrniuzl2OdiCD1ftad+WLA/q04SzvgeWCQAHV4='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'pool', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'journal', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'clyso-debian-13', 'teuthology_repo': 'https://github.com/clyso/teuthology', 'teuthology_sha1': '1c580df7a9c7c2aadc272da296344fd99f27c444', 'timestamp': '2026-03-20_22:04:26', 'tube': 'vps', 'user': 'kyr', 'verbose': False, 'worker_log': '/home/teuthos/.teuthology/dispatcher/dispatcher.vps.2366871'} 2026-03-24T02:13:36.593 DEBUG:teuthology.orchestra.run.vm06:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-03-24T02:13:36.594 DEBUG:teuthology.orchestra.run.vm07:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-03-24T02:13:36.598 DEBUG:teuthology.orchestra.run.vm06:> sudo install -d -m0777 -- /var/run/ceph 2026-03-24T02:13:36.640 DEBUG:teuthology.orchestra.run.vm07:> sudo install -d -m0777 -- /var/run/ceph 2026-03-24T02:13:36.648 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:36.648 DEBUG:teuthology.orchestra.run.vm07:> dd if=/scratch_devs of=/dev/stdout 2026-03-24T02:13:36.690 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-03-24T02:13:36.690 DEBUG:teuthology.orchestra.run.vm07:> stat /dev/vg_nvme/lv_1 2026-03-24T02:13:36.735 INFO:teuthology.orchestra.run.vm07.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-03-24T02:13:36.735 INFO:teuthology.orchestra.run.vm07.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-24T02:13:36.735 INFO:teuthology.orchestra.run.vm07.stdout:Device: 5h/5d Inode: 793 Links: 1 2026-03-24T02:13:36.735 INFO:teuthology.orchestra.run.vm07.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-24T02:13:36.735 INFO:teuthology.orchestra.run.vm07.stdout:Access: 2026-03-24 02:11:52.880097000 +0000 2026-03-24T02:13:36.735 INFO:teuthology.orchestra.run.vm07.stdout:Modify: 2026-03-24 02:11:52.756097000 +0000 2026-03-24T02:13:36.735 INFO:teuthology.orchestra.run.vm07.stdout:Change: 2026-03-24 02:11:52.756097000 +0000 2026-03-24T02:13:36.735 INFO:teuthology.orchestra.run.vm07.stdout: Birth: - 2026-03-24T02:13:36.735 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-03-24T02:13:36.782 INFO:teuthology.orchestra.run.vm07.stderr:1+0 records in 2026-03-24T02:13:36.782 INFO:teuthology.orchestra.run.vm07.stderr:1+0 records out 2026-03-24T02:13:36.782 INFO:teuthology.orchestra.run.vm07.stderr:512 bytes copied, 0.000179235 s, 2.9 MB/s 2026-03-24T02:13:36.782 DEBUG:teuthology.orchestra.run.vm07:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-03-24T02:13:36.827 DEBUG:teuthology.orchestra.run.vm07:> stat /dev/vg_nvme/lv_2 2026-03-24T02:13:36.870 INFO:teuthology.orchestra.run.vm07.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-03-24T02:13:36.870 INFO:teuthology.orchestra.run.vm07.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-24T02:13:36.870 INFO:teuthology.orchestra.run.vm07.stdout:Device: 5h/5d Inode: 825 Links: 1 2026-03-24T02:13:36.870 INFO:teuthology.orchestra.run.vm07.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-24T02:13:36.870 INFO:teuthology.orchestra.run.vm07.stdout:Access: 2026-03-24 02:11:53.040097000 +0000 2026-03-24T02:13:36.870 INFO:teuthology.orchestra.run.vm07.stdout:Modify: 2026-03-24 02:11:53.036097000 +0000 2026-03-24T02:13:36.870 INFO:teuthology.orchestra.run.vm07.stdout:Change: 2026-03-24 02:11:53.036097000 +0000 2026-03-24T02:13:36.870 INFO:teuthology.orchestra.run.vm07.stdout: Birth: - 2026-03-24T02:13:36.870 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-03-24T02:13:36.917 INFO:teuthology.orchestra.run.vm07.stderr:1+0 records in 2026-03-24T02:13:36.917 INFO:teuthology.orchestra.run.vm07.stderr:1+0 records out 2026-03-24T02:13:36.917 INFO:teuthology.orchestra.run.vm07.stderr:512 bytes copied, 0.000168486 s, 3.0 MB/s 2026-03-24T02:13:36.918 DEBUG:teuthology.orchestra.run.vm07:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-03-24T02:13:36.963 DEBUG:teuthology.orchestra.run.vm07:> stat /dev/vg_nvme/lv_3 2026-03-24T02:13:37.006 INFO:teuthology.orchestra.run.vm07.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-03-24T02:13:37.006 INFO:teuthology.orchestra.run.vm07.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-24T02:13:37.006 INFO:teuthology.orchestra.run.vm07.stdout:Device: 5h/5d Inode: 855 Links: 1 2026-03-24T02:13:37.006 INFO:teuthology.orchestra.run.vm07.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-24T02:13:37.006 INFO:teuthology.orchestra.run.vm07.stdout:Access: 2026-03-24 02:11:53.480097000 +0000 2026-03-24T02:13:37.006 INFO:teuthology.orchestra.run.vm07.stdout:Modify: 2026-03-24 02:11:53.348097000 +0000 2026-03-24T02:13:37.006 INFO:teuthology.orchestra.run.vm07.stdout:Change: 2026-03-24 02:11:53.348097000 +0000 2026-03-24T02:13:37.006 INFO:teuthology.orchestra.run.vm07.stdout: Birth: - 2026-03-24T02:13:37.006 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-03-24T02:13:37.053 INFO:teuthology.orchestra.run.vm07.stderr:1+0 records in 2026-03-24T02:13:37.053 INFO:teuthology.orchestra.run.vm07.stderr:1+0 records out 2026-03-24T02:13:37.053 INFO:teuthology.orchestra.run.vm07.stderr:512 bytes copied, 0.000178303 s, 2.9 MB/s 2026-03-24T02:13:37.054 DEBUG:teuthology.orchestra.run.vm07:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-03-24T02:13:37.099 DEBUG:teuthology.orchestra.run.vm07:> stat /dev/vg_nvme/lv_4 2026-03-24T02:13:37.146 INFO:teuthology.orchestra.run.vm07.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-03-24T02:13:37.146 INFO:teuthology.orchestra.run.vm07.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-24T02:13:37.146 INFO:teuthology.orchestra.run.vm07.stdout:Device: 5h/5d Inode: 887 Links: 1 2026-03-24T02:13:37.146 INFO:teuthology.orchestra.run.vm07.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-24T02:13:37.146 INFO:teuthology.orchestra.run.vm07.stdout:Access: 2026-03-24 02:11:53.644097000 +0000 2026-03-24T02:13:37.146 INFO:teuthology.orchestra.run.vm07.stdout:Modify: 2026-03-24 02:11:53.640097000 +0000 2026-03-24T02:13:37.146 INFO:teuthology.orchestra.run.vm07.stdout:Change: 2026-03-24 02:11:53.640097000 +0000 2026-03-24T02:13:37.146 INFO:teuthology.orchestra.run.vm07.stdout: Birth: - 2026-03-24T02:13:37.147 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-03-24T02:13:37.193 INFO:teuthology.orchestra.run.vm07.stderr:1+0 records in 2026-03-24T02:13:37.193 INFO:teuthology.orchestra.run.vm07.stderr:1+0 records out 2026-03-24T02:13:37.193 INFO:teuthology.orchestra.run.vm07.stderr:512 bytes copied, 0.000174467 s, 2.9 MB/s 2026-03-24T02:13:37.194 DEBUG:teuthology.orchestra.run.vm07:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-03-24T02:13:37.239 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-24T02:13:37.239 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@vm07.local'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-03-24T02:13:37.239 INFO:tasks.ceph:Generating config... 2026-03-24T02:13:37.239 INFO:tasks.ceph:[client] debug rbd = 20 2026-03-24T02:13:37.239 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-03-24T02:13:37.239 INFO:tasks.ceph:[client] log to stderr = False 2026-03-24T02:13:37.239 INFO:tasks.ceph:[client] rbd default features = 125 2026-03-24T02:13:37.239 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2026-03-24T02:13:37.239 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:37.239 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:37.239 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:37.239 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:37.239 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:37.240 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:37.240 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:37.240 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:37.240 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-03-24T02:13:37.240 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-03-24T02:13:37.240 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-03-24T02:13:37.240 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-03-24T02:13:37.240 INFO:tasks.ceph:[mgr] debug ms = 1 2026-03-24T02:13:37.240 INFO:tasks.ceph:[mon] debug mon = 20 2026-03-24T02:13:37.240 INFO:tasks.ceph:[mon] debug ms = 1 2026-03-24T02:13:37.240 INFO:tasks.ceph:[mon] debug paxos = 20 2026-03-24T02:13:37.240 INFO:tasks.ceph:[osd] bluefs allocator = hybrid 2026-03-24T02:13:37.240 INFO:tasks.ceph:[osd] bluestore allocator = hybrid 2026-03-24T02:13:37.240 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-03-24T02:13:37.240 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-03-24T02:13:37.240 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-03-24T02:13:37.240 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-03-24T02:13:37.240 INFO:tasks.ceph:[osd] debug ms = 1 2026-03-24T02:13:37.240 INFO:tasks.ceph:[osd] debug osd = 20 2026-03-24T02:13:37.240 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-03-24T02:13:37.240 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-03-24T02:13:37.240 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-03-24T02:13:37.240 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-03-24T02:13:37.240 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-03-24T02:13:37.240 INFO:tasks.ceph:[osd] osd mclock iops capacity threshold hdd = 49000 2026-03-24T02:13:37.240 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-03-24T02:13:37.240 INFO:tasks.ceph:Setting up cluster2.mon.a... 2026-03-24T02:13:37.240 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2026-03-24T02:13:37.297 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/cluster2.keyring 2026-03-24T02:13:37.299 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2026-03-24T02:13:37.360 DEBUG:teuthology.orchestra.run.vm07:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-03-24T02:13:37.411 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '192.168.123.107')] 2026-03-24T02:13:37.411 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '192.168.123.107', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-03-24T02:13:37.412 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:37.412 DEBUG:teuthology.orchestra.run.vm07:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-03-24T02:13:37.455 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 192.168.123.107 --print /home/ubuntu/cephtest/cluster2.monmap 2026-03-24T02:13:37.513 INFO:teuthology.orchestra.run.vm07.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2026-03-24T02:13:37.513 INFO:teuthology.orchestra.run.vm07.stdout:monmaptool: generated fsid 91ca8768-ba68-432d-bc1f-efeea617a922 2026-03-24T02:13:37.513 INFO:teuthology.orchestra.run.vm07.stdout:setting min_mon_release = tentacle 2026-03-24T02:13:37.513 INFO:teuthology.orchestra.run.vm07.stdout:epoch 0 2026-03-24T02:13:37.513 INFO:teuthology.orchestra.run.vm07.stdout:fsid 91ca8768-ba68-432d-bc1f-efeea617a922 2026-03-24T02:13:37.513 INFO:teuthology.orchestra.run.vm07.stdout:last_changed 2026-03-24T02:13:37.514464+0000 2026-03-24T02:13:37.513 INFO:teuthology.orchestra.run.vm07.stdout:created 2026-03-24T02:13:37.514464+0000 2026-03-24T02:13:37.513 INFO:teuthology.orchestra.run.vm07.stdout:min_mon_release 20 (tentacle) 2026-03-24T02:13:37.513 INFO:teuthology.orchestra.run.vm07.stdout:election_strategy: 1 2026-03-24T02:13:37.513 INFO:teuthology.orchestra.run.vm07.stdout:0: [v2:192.168.123.107:3300/0,v1:192.168.123.107:6789/0] mon.a 2026-03-24T02:13:37.513 INFO:teuthology.orchestra.run.vm07.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2026-03-24T02:13:37.514 DEBUG:teuthology.orchestra.run.vm07:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-03-24T02:13:37.559 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID 91ca8768-ba68-432d-bc1f-efeea617a922... 2026-03-24T02:13:37.559 DEBUG:teuthology.orchestra.run.vm06:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-03-24T02:13:37.560 DEBUG:teuthology.orchestra.run.vm07:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout:[global] 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: chdir = "" 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: auth supported = cephx 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: filestore xattr use omap = true 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: mon clock drift allowed = 1.000 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: osd crush chooseleaf type = 0 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: auth debug = true 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: ms die on old message = true 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: ms die on bug = true 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: mon max pg per osd = 10000 # >= luminous 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: mon pg warn max object skew = 0 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: # disable pg_autoscaler by default for new pools 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: osd_pool_default_pg_autoscale_mode = off 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: osd pool default size = 2 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: mon osd allow primary affinity = true 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: mon osd allow pg remap = true 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: mon warn on legacy crush tunables = false 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: mon warn on crush straw calc version zero = false 2026-03-24T02:13:37.601 INFO:teuthology.orchestra.run.vm06.stdout: mon warn on no sortbitwise = false 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: mon warn on osd down out interval zero = false 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: mon warn on too few osds = false 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: mon_warn_on_pool_no_redundancy = false 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: mon_allow_pool_size_one = true 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: osd default data pool replay window = 5 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: mon allow pool delete = true 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: mon cluster log file level = debug 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: debug asserts on shutdown = true 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: mon health detail to clog = false 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: mon host = 192.168.123.107 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: mon client directed command retry = 5 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: mon warn on pool no app = False 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: ms inject socket failures = 5000 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: fsid = 91ca8768-ba68-432d-bc1f-efeea617a922 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout:[osd] 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: osd journal size = 100 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: osd scrub load threshold = 5.0 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: osd scrub max interval = 600 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: osd mclock profile = high_recovery_ops 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: osd mclock skip benchmark = true 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: osd recover clone overlap = true 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: osd recovery max chunk = 1048576 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: osd debug shutdown = true 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: osd debug op order = true 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: osd debug verify stray on activate = true 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: osd debug trim objects = true 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: osd open classes on start = true 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: osd debug pg log writeout = true 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: osd deep scrub update digest min age = 30 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: osd map max advance = 10 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: journal zero on create = true 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: filestore ondisk finisher threads = 3 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: filestore apply finisher threads = 3 2026-03-24T02:13:37.602 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: bdev debug aio = true 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: osd debug misdirected ops = true 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: bluefs allocator = hybrid 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: bluestore allocator = hybrid 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: bluestore block size = 96636764160 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: bluestore fsck on mount = True 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: debug bluefs = 1/20 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: debug bluestore = 1/20 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: debug ms = 1 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: debug osd = 20 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: debug rocksdb = 4/10 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: mon osd backfillfull_ratio = 0.85 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: mon osd full ratio = 0.9 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: mon osd nearfull ratio = 0.8 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: osd failsafe full ratio = 0.95 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: osd mclock iops capacity threshold hdd = 49000 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: osd objectstore = bluestore 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout:[mgr] 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: debug ms = 1 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: debug mgr = 20 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: debug mon = 20 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: debug auth = 20 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: mon reweight min pgs per osd = 4 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: mon reweight min bytes per osd = 10 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: mgr/telemetry/nag = false 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout:[mon] 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: debug ms = 1 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: debug mon = 20 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: debug paxos = 20 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: debug auth = 20 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: mon data avail warn = 5 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: mon mgr mkfs grace = 240 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: mon reweight min pgs per osd = 4 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: mon osd reporter subtree level = osd 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: mon osd prime pg temp = true 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: mon reweight min bytes per osd = 10 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: auth mon ticket ttl = 660 # 11m 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: auth service ticket ttl = 240 # 4m 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: # don't complain about insecure global_id in the test suite 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.603 INFO:teuthology.orchestra.run.vm06.stdout: # 1m isn't quite enough 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: mon_down_mkfs_grace = 2m 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: mon_warn_on_filestore_osds = false 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout:[client] 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: rgw cache enabled = true 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: rgw enable ops log = true 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: rgw enable usage log = true 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: debug rbd = 20 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: debug rbd_mirror = 15 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: log to stderr = False 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: rbd default features = 125 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: rbd mirror image policy type = simple 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout:[mon.a] 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout:[client.mirror.0] 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout:[client.mirror.1] 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout:[client.mirror.2] 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout:[client.mirror.3] 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:37.604 INFO:teuthology.orchestra.run.vm06.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:37.618 INFO:teuthology.orchestra.run.vm07.stdout:[global] 2026-03-24T02:13:37.618 INFO:teuthology.orchestra.run.vm07.stdout: chdir = "" 2026-03-24T02:13:37.618 INFO:teuthology.orchestra.run.vm07.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-03-24T02:13:37.618 INFO:teuthology.orchestra.run.vm07.stdout: auth supported = cephx 2026-03-24T02:13:37.618 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.618 INFO:teuthology.orchestra.run.vm07.stdout: filestore xattr use omap = true 2026-03-24T02:13:37.618 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon clock drift allowed = 1.000 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd crush chooseleaf type = 0 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: auth debug = true 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: ms die on old message = true 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: ms die on bug = true 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon max pg per osd = 10000 # >= luminous 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon pg warn max object skew = 0 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: # disable pg_autoscaler by default for new pools 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd_pool_default_pg_autoscale_mode = off 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd pool default size = 2 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon osd allow primary affinity = true 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon osd allow pg remap = true 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on legacy crush tunables = false 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on crush straw calc version zero = false 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on no sortbitwise = false 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on osd down out interval zero = false 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on too few osds = false 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_pool_no_redundancy = false 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon_allow_pool_size_one = true 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd default data pool replay window = 5 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon allow pool delete = true 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon cluster log file level = debug 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: debug asserts on shutdown = true 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon health detail to clog = false 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon host = 192.168.123.107 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon client directed command retry = 5 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on pool no app = False 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: ms inject socket failures = 5000 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: fsid = 91ca8768-ba68-432d-bc1f-efeea617a922 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout:[osd] 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd journal size = 100 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd scrub load threshold = 5.0 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd scrub max interval = 600 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd mclock profile = high_recovery_ops 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd mclock skip benchmark = true 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd recover clone overlap = true 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd recovery max chunk = 1048576 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd debug shutdown = true 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd debug op order = true 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd debug verify stray on activate = true 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd debug trim objects = true 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd open classes on start = true 2026-03-24T02:13:37.619 INFO:teuthology.orchestra.run.vm07.stdout: osd debug pg log writeout = true 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: osd deep scrub update digest min age = 30 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: osd map max advance = 10 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: journal zero on create = true 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: filestore ondisk finisher threads = 3 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: filestore apply finisher threads = 3 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: bdev debug aio = true 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: osd debug misdirected ops = true 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: bluefs allocator = hybrid 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: bluestore allocator = hybrid 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: bluestore block size = 96636764160 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: bluestore fsck on mount = True 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: debug bluefs = 1/20 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: debug bluestore = 1/20 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: debug ms = 1 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: debug osd = 20 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: debug rocksdb = 4/10 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: mon osd backfillfull_ratio = 0.85 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: mon osd full ratio = 0.9 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: mon osd nearfull ratio = 0.8 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: osd failsafe full ratio = 0.95 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: osd mclock iops capacity threshold hdd = 49000 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: osd objectstore = bluestore 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout:[mgr] 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: debug ms = 1 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: debug mgr = 20 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: debug mon = 20 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: debug auth = 20 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: mon reweight min pgs per osd = 4 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: mon reweight min bytes per osd = 10 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: mgr/telemetry/nag = false 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout:[mon] 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: debug ms = 1 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: debug mon = 20 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: debug paxos = 20 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: debug auth = 20 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: mon data avail warn = 5 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: mon mgr mkfs grace = 240 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: mon reweight min pgs per osd = 4 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: mon osd reporter subtree level = osd 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: mon osd prime pg temp = true 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: mon reweight min bytes per osd = 10 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: auth mon ticket ttl = 660 # 11m 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: auth service ticket ttl = 240 # 4m 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: # don't complain about insecure global_id in the test suite 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: # 1m isn't quite enough 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: mon_down_mkfs_grace = 2m 2026-03-24T02:13:37.620 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_filestore_osds = false 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout:[client] 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: rgw cache enabled = true 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: rgw enable ops log = true 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: rgw enable usage log = true 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: debug rbd = 20 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: debug rbd_mirror = 15 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: log to stderr = False 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: rbd default features = 125 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: rbd mirror image policy type = simple 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout:[mon.a] 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.0] 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.1] 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.2] 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.3] 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-24T02:13:37.621 INFO:teuthology.orchestra.run.vm07.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-24T02:13:37.626 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2026-03-24T02:13:37.626 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2026-03-24T02:13:37.694 INFO:tasks.ceph:Copying monmap to all nodes... 2026-03-24T02:13:37.694 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:37.694 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2026-03-24T02:13:37.739 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:37.739 DEBUG:teuthology.orchestra.run.vm07:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2026-03-24T02:13:37.782 INFO:tasks.ceph:Sending monmap to node ubuntu@vm06.local 2026-03-24T02:13:37.782 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:37.782 DEBUG:teuthology.orchestra.run.vm06:> sudo dd of=/etc/ceph/cluster2.keyring 2026-03-24T02:13:37.782 DEBUG:teuthology.orchestra.run.vm06:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-03-24T02:13:37.797 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:37.797 DEBUG:teuthology.orchestra.run.vm06:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-03-24T02:13:37.842 INFO:tasks.ceph:Sending monmap to node ubuntu@vm07.local 2026-03-24T02:13:37.842 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:37.842 DEBUG:teuthology.orchestra.run.vm07:> sudo dd of=/etc/ceph/cluster2.keyring 2026-03-24T02:13:37.842 DEBUG:teuthology.orchestra.run.vm07:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-03-24T02:13:37.854 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:37.860 DEBUG:teuthology.orchestra.run.vm07:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-03-24T02:13:37.898 INFO:tasks.ceph:Setting up mon nodes... 2026-03-24T02:13:37.898 INFO:tasks.ceph:Setting up mgr nodes... 2026-03-24T02:13:37.898 DEBUG:teuthology.orchestra.run.vm06:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2026-03-24T02:13:37.924 INFO:teuthology.orchestra.run.vm06.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2026-03-24T02:13:37.926 INFO:tasks.ceph:Setting up mds nodes... 2026-03-24T02:13:37.926 INFO:tasks.ceph_client:Setting up client nodes... 2026-03-24T02:13:37.926 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2026-03-24T02:13:37.987 INFO:teuthology.orchestra.run.vm06.stdout:creating /etc/ceph/cluster2.client.0.keyring 2026-03-24T02:13:37.994 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2026-03-24T02:13:38.014 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2026-03-24T02:13:38.021 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2026-03-24T02:13:38.082 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2026-03-24T02:13:38.089 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2026-03-24T02:13:38.150 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2026-03-24T02:13:38.157 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2026-03-24T02:13:38.218 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2026-03-24T02:13:38.226 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2026-03-24T02:13:38.287 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2026-03-24T02:13:38.294 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2026-03-24T02:13:38.356 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2026-03-24T02:13:38.363 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2026-03-24T02:13:38.427 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2026-03-24T02:13:38.434 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2026-03-24T02:13:38.494 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2026-03-24T02:13:38.501 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-03-24T02:13:38.501 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@vm06.local'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@vm07.local'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-03-24T02:13:38.501 DEBUG:teuthology.orchestra.run.vm07:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2026-03-24T02:13:38.551 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-24T02:13:38.551 INFO:tasks.ceph:role: cluster2.osd.0 2026-03-24T02:13:38.551 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@vm07.local 2026-03-24T02:13:38.552 DEBUG:teuthology.orchestra.run.vm07:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-03-24T02:13:38.602 INFO:teuthology.orchestra.run.vm07.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=1310464 blks 2026-03-24T02:13:38.602 INFO:teuthology.orchestra.run.vm07.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-24T02:13:38.602 INFO:teuthology.orchestra.run.vm07.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-24T02:13:38.602 INFO:teuthology.orchestra.run.vm07.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-03-24T02:13:38.602 INFO:teuthology.orchestra.run.vm07.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-24T02:13:38.602 INFO:teuthology.orchestra.run.vm07.stdout: = sunit=0 swidth=0 blks 2026-03-24T02:13:38.602 INFO:teuthology.orchestra.run.vm07.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-24T02:13:38.603 INFO:teuthology.orchestra.run.vm07.stdout:log =internal log bsize=4096 blocks=2560, version=2 2026-03-24T02:13:38.603 INFO:teuthology.orchestra.run.vm07.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-24T02:13:38.603 INFO:teuthology.orchestra.run.vm07.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-24T02:13:38.607 INFO:teuthology.orchestra.run.vm07.stdout:Discarding blocks...Done. 2026-03-24T02:13:38.609 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@vm07.local -o noatime 2026-03-24T02:13:38.609 DEBUG:teuthology.orchestra.run.vm07:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2026-03-24T02:13:38.702 DEBUG:teuthology.orchestra.run.vm07:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2026-03-24T02:13:38.708 INFO:teuthology.orchestra.run.vm07.stderr:sudo: /sbin/restorecon: command not found 2026-03-24T02:13:38.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-24T02:13:38.709 DEBUG:teuthology.orchestra.run.vm07:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2026-03-24T02:13:38.760 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-24T02:13:38.760 INFO:tasks.ceph:role: cluster2.osd.1 2026-03-24T02:13:38.760 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@vm07.local 2026-03-24T02:13:38.760 DEBUG:teuthology.orchestra.run.vm07:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-03-24T02:13:38.811 INFO:teuthology.orchestra.run.vm07.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=1310464 blks 2026-03-24T02:13:38.811 INFO:teuthology.orchestra.run.vm07.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-24T02:13:38.811 INFO:teuthology.orchestra.run.vm07.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-24T02:13:38.811 INFO:teuthology.orchestra.run.vm07.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-03-24T02:13:38.811 INFO:teuthology.orchestra.run.vm07.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-24T02:13:38.811 INFO:teuthology.orchestra.run.vm07.stdout: = sunit=0 swidth=0 blks 2026-03-24T02:13:38.811 INFO:teuthology.orchestra.run.vm07.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-24T02:13:38.811 INFO:teuthology.orchestra.run.vm07.stdout:log =internal log bsize=4096 blocks=2560, version=2 2026-03-24T02:13:38.811 INFO:teuthology.orchestra.run.vm07.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-24T02:13:38.811 INFO:teuthology.orchestra.run.vm07.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-24T02:13:38.858 INFO:teuthology.orchestra.run.vm07.stdout:Discarding blocks...Done. 2026-03-24T02:13:38.859 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@vm07.local -o noatime 2026-03-24T02:13:38.859 DEBUG:teuthology.orchestra.run.vm07:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2026-03-24T02:13:38.910 DEBUG:teuthology.orchestra.run.vm07:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2026-03-24T02:13:38.957 INFO:teuthology.orchestra.run.vm07.stderr:sudo: /sbin/restorecon: command not found 2026-03-24T02:13:38.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-24T02:13:38.958 DEBUG:teuthology.orchestra.run.vm07:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2026-03-24T02:13:39.006 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-24T02:13:39.006 INFO:tasks.ceph:role: cluster2.osd.2 2026-03-24T02:13:39.006 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@vm07.local 2026-03-24T02:13:39.006 DEBUG:teuthology.orchestra.run.vm07:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-03-24T02:13:39.055 INFO:teuthology.orchestra.run.vm07.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=1310464 blks 2026-03-24T02:13:39.055 INFO:teuthology.orchestra.run.vm07.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-24T02:13:39.055 INFO:teuthology.orchestra.run.vm07.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-24T02:13:39.055 INFO:teuthology.orchestra.run.vm07.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-03-24T02:13:39.055 INFO:teuthology.orchestra.run.vm07.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-24T02:13:39.055 INFO:teuthology.orchestra.run.vm07.stdout: = sunit=0 swidth=0 blks 2026-03-24T02:13:39.055 INFO:teuthology.orchestra.run.vm07.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-24T02:13:39.055 INFO:teuthology.orchestra.run.vm07.stdout:log =internal log bsize=4096 blocks=2560, version=2 2026-03-24T02:13:39.055 INFO:teuthology.orchestra.run.vm07.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-24T02:13:39.055 INFO:teuthology.orchestra.run.vm07.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-24T02:13:39.060 INFO:teuthology.orchestra.run.vm07.stdout:Discarding blocks...Done. 2026-03-24T02:13:39.061 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@vm07.local -o noatime 2026-03-24T02:13:39.061 DEBUG:teuthology.orchestra.run.vm07:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2026-03-24T02:13:39.115 DEBUG:teuthology.orchestra.run.vm07:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2026-03-24T02:13:39.164 INFO:teuthology.orchestra.run.vm07.stderr:sudo: /sbin/restorecon: command not found 2026-03-24T02:13:39.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-24T02:13:39.164 DEBUG:teuthology.orchestra.run.vm07:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-03-24T02:13:39.230 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-24T02:13:39.225+0000 7f69fd49ba40 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2026-03-24T02:13:39.231 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-24T02:13:39.225+0000 7f69fd49ba40 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2026-03-24T02:13:39.231 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-24T02:13:39.225+0000 7f69fd49ba40 -1 bdev(0x55bca8d9b800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2026-03-24T02:13:39.231 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-24T02:13:39.225+0000 7f69fd49ba40 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2026-03-24T02:13:40.264 DEBUG:teuthology.orchestra.run.vm07:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2026-03-24T02:13:40.315 DEBUG:teuthology.orchestra.run.vm07:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-03-24T02:13:40.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-24T02:13:40.373+0000 7fbbbde14a40 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2026-03-24T02:13:40.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-24T02:13:40.373+0000 7fbbbde14a40 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2026-03-24T02:13:40.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-24T02:13:40.373+0000 7fbbbde14a40 -1 bdev(0x557523fa7800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2026-03-24T02:13:40.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-24T02:13:40.373+0000 7fbbbde14a40 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2026-03-24T02:13:41.256 DEBUG:teuthology.orchestra.run.vm07:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2026-03-24T02:13:41.307 DEBUG:teuthology.orchestra.run.vm07:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-03-24T02:13:41.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-24T02:13:41.365+0000 7f66082a8a40 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2026-03-24T02:13:41.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-24T02:13:41.365+0000 7f66082a8a40 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2026-03-24T02:13:41.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-24T02:13:41.365+0000 7f66082a8a40 -1 bdev(0x56010c20d800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2026-03-24T02:13:41.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-24T02:13:41.365+0000 7f66082a8a40 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2026-03-24T02:13:42.347 DEBUG:teuthology.orchestra.run.vm07:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2026-03-24T02:13:42.396 INFO:tasks.ceph:Reading keys from all nodes... 2026-03-24T02:13:42.396 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:42.396 DEBUG:teuthology.orchestra.run.vm06:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2026-03-24T02:13:42.406 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:42.406 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2026-03-24T02:13:42.451 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:42.452 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2026-03-24T02:13:42.508 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:42.508 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2026-03-24T02:13:42.564 DEBUG:teuthology.orchestra.run.vm06:> set -ex 2026-03-24T02:13:42.564 DEBUG:teuthology.orchestra.run.vm06:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2026-03-24T02:13:42.568 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:42.568 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2026-03-24T02:13:42.611 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:42.611 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2026-03-24T02:13:42.656 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:42.656 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2026-03-24T02:13:42.703 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:42.703 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2026-03-24T02:13:42.747 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:42.747 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2026-03-24T02:13:42.791 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:42.791 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2026-03-24T02:13:42.835 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:42.835 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2026-03-24T02:13:42.879 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:42.879 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2026-03-24T02:13:42.923 INFO:tasks.ceph:Adding keys to all mons... 2026-03-24T02:13:42.923 DEBUG:teuthology.orchestra.run.vm07:> sudo tee -a /etc/ceph/cluster2.keyring 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout:[mgr.x] 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBR88FpH+ojNxAAYIRn/5GRs5K51ARkRPSMWA== 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout:[osd.0] 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBT88FpixfWDRAACHycVm+jJdxtdcM140th9g== 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout:[osd.1] 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBU88Fpr/OvFhAAz9+wA/135JgZcK7MmFUmEw== 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout:[osd.2] 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBV88FpJ2oMFhAAEet/BzOaSSk1eYWbmAPeNQ== 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout:[client.0] 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBR88FpHVXpOhAAP+0pevWqcuZrER1TIxJC3Q== 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror] 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBS88FpX+HrABAAeI6XlW8Y+zinDD9KOq4Blg== 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.0] 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBS88FpFqoEBRAAiWzcM4LV0Ow+JbKs9KY1VA== 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.1] 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBS88FpyLoNCRAAqlwDI+7jCTFHZe8bnBig3A== 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.2] 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBS88FppxccDRAA6dYCZqy1MhT+ZmqRrR5hYw== 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.3] 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBS88Fp4FUwERAAW4sC/UfOh+zPZQzJd7hOOQ== 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.4] 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBS88FpjcVOFRAAK+3YzHNNWq3HK4dT0U3hhQ== 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.5] 2026-03-24T02:13:42.970 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBS88Fp8GSTGRAA9DHLQprDI5pBU1yES+Y8uw== 2026-03-24T02:13:42.971 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.6] 2026-03-24T02:13:42.971 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBS88Fpu6CHHRAA9xpMkmrvgzzpy06hRGdAvA== 2026-03-24T02:13:42.971 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-03-24T02:13:43.037 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-24T02:13:43.102 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-24T02:13:43.169 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-24T02:13:43.236 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:43.303 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:43.367 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:43.434 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:43.502 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:43.569 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:43.642 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:43.707 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:43.772 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-24T02:13:43.840 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-03-24T02:13:43.840 DEBUG:teuthology.orchestra.run.vm07:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2026-03-24T02:13:43.891 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2026-03-24T02:13:43.970 DEBUG:teuthology.orchestra.run.vm07:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2026-03-24T02:13:44.020 DEBUG:teuthology.orchestra.run.vm07:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2026-03-24T02:13:44.066 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2026-03-24T02:13:44.067 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2026-03-24T02:13:44.067 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2026-03-24T02:13:44.108 INFO:tasks.ceph.cluster2.mon.a:Started 2026-03-24T02:13:44.108 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2026-03-24T02:13:44.109 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2026-03-24T02:13:44.109 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2026-03-24T02:13:44.110 INFO:tasks.ceph.cluster2.mgr.x:Started 2026-03-24T02:13:44.110 DEBUG:tasks.ceph:set 0 configs 2026-03-24T02:13:44.110 DEBUG:teuthology.orchestra.run.vm07:> sudo ceph --cluster cluster2 config dump 2026-03-24T02:13:44.817 INFO:teuthology.orchestra.run.vm07.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-03-24T02:13:44.830 INFO:tasks.ceph:Setting crush tunables to default 2026-03-24T02:13:44.830 DEBUG:teuthology.orchestra.run.vm07:> sudo ceph --cluster cluster2 osd crush tunables default 2026-03-24T02:13:44.941 INFO:teuthology.orchestra.run.vm07.stderr:adjusted tunables profile to default 2026-03-24T02:13:44.957 INFO:tasks.ceph:check_enable_crimson: False 2026-03-24T02:13:44.957 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2026-03-24T02:13:44.957 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:44.957 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2026-03-24T02:13:44.966 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:44.966 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2026-03-24T02:13:45.016 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:13:45.017 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2026-03-24T02:13:45.070 DEBUG:teuthology.orchestra.run.vm07:> sudo ceph --cluster cluster2 osd new f02a1366-0e32-4132-9124-8b60215ba403 0 2026-03-24T02:13:45.233 INFO:teuthology.orchestra.run.vm07.stdout:0 2026-03-24T02:13:45.247 DEBUG:teuthology.orchestra.run.vm07:> sudo ceph --cluster cluster2 osd new c3c9032d-80f2-481b-b324-d91af15845ca 1 2026-03-24T02:13:45.363 INFO:teuthology.orchestra.run.vm07.stdout:1 2026-03-24T02:13:45.379 DEBUG:teuthology.orchestra.run.vm07:> sudo ceph --cluster cluster2 osd new bca91acd-ce62-44d0-b940-2dcc1877d928 2 2026-03-24T02:13:45.593 INFO:teuthology.orchestra.run.vm07.stdout:2 2026-03-24T02:13:45.607 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2026-03-24T02:13:45.607 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2026-03-24T02:13:45.608 INFO:tasks.ceph.cluster2.osd.0:Started 2026-03-24T02:13:45.608 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2026-03-24T02:13:45.608 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2026-03-24T02:13:45.609 INFO:tasks.ceph.cluster2.osd.1:Started 2026-03-24T02:13:45.609 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2026-03-24T02:13:45.609 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2026-03-24T02:13:45.610 INFO:tasks.ceph.cluster2.osd.2:Started 2026-03-24T02:13:45.610 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-03-24T02:13:45.778 INFO:tasks.ceph.cluster2.mgr.x.vm06.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-03-24T02:13:45.778 INFO:tasks.ceph.cluster2.mgr.x.vm06.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-03-24T02:13:45.778 INFO:tasks.ceph.cluster2.mgr.x.vm06.stderr: from numpy import show_config as show_numpy_config 2026-03-24T02:13:45.921 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:45.921 INFO:teuthology.orchestra.run.vm07.stdout:{"epoch":5,"fsid":"91ca8768-ba68-432d-bc1f-efeea617a922","created":"2026-03-24T02:13:44.473145+0000","modified":"2026-03-24T02:13:45.559913+0000","last_up_change":"0.000000","last_in_change":"2026-03-24T02:13:45.559913+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"f02a1366-0e32-4132-9124-8b60215ba403","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"c3c9032d-80f2-481b-b324-d91af15845ca","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"bca91acd-ce62-44d0-b940-2dcc1877d928","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-24T02:13:45.924 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:13:45.921+0000 7f086bf34a40 -1 Falling back to public interface 2026-03-24T02:13:45.934 INFO:tasks.ceph.ceph_manager.cluster2:[] 2026-03-24T02:13:45.934 INFO:tasks.ceph:Waiting for OSDs to come up 2026-03-24T02:13:45.968 INFO:tasks.ceph.cluster2.osd.1.vm07.stderr:2026-03-24T02:13:45.965+0000 7f2e19efca40 -1 Falling back to public interface 2026-03-24T02:13:45.984 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:13:45.981+0000 7f7749710a40 -1 Falling back to public interface 2026-03-24T02:13:46.236 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-03-24T02:13:46.356 INFO:teuthology.misc.health.vm07.stdout: 2026-03-24T02:13:46.356 INFO:teuthology.misc.health.vm07.stdout:{"epoch":5,"fsid":"91ca8768-ba68-432d-bc1f-efeea617a922","created":"2026-03-24T02:13:44.473145+0000","modified":"2026-03-24T02:13:45.559913+0000","last_up_change":"0.000000","last_in_change":"2026-03-24T02:13:45.559913+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"f02a1366-0e32-4132-9124-8b60215ba403","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"c3c9032d-80f2-481b-b324-d91af15845ca","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"bca91acd-ce62-44d0-b940-2dcc1877d928","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-24T02:13:46.373 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-03-24T02:13:46.381 INFO:tasks.ceph.cluster2.mgr.x.vm06.stderr:Failed to import NVMeoFClient and related components: cannot import name 'NVMeoFClient' from 'dashboard.services.nvmeof_client' (/usr/share/ceph/mgr/dashboard/services/nvmeof_client.py) 2026-03-24T02:13:46.683 INFO:tasks.ceph.cluster2.osd.1.vm07.stderr:2026-03-24T02:13:46.677+0000 7f2e19efca40 -1 osd.1 0 log_to_monitors true 2026-03-24T02:13:46.699 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:13:46.693+0000 7f086bf34a40 -1 osd.2 0 log_to_monitors true 2026-03-24T02:13:46.700 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:13:46.697+0000 7f7749710a40 -1 osd.0 0 log_to_monitors true 2026-03-24T02:13:48.001 INFO:tasks.ceph.cluster2.mgr.x.vm06.stderr:2026-03-24T02:13:47.998+0000 7fde2588d640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-03-24T02:13:48.780 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:13:48.777+0000 7f77456b9640 -1 osd.0 0 waiting for initial osdmap 2026-03-24T02:13:48.780 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:13:48.777+0000 7f0867edd640 -1 osd.2 0 waiting for initial osdmap 2026-03-24T02:13:48.780 INFO:tasks.ceph.cluster2.osd.1.vm07.stderr:2026-03-24T02:13:48.777+0000 7f2e166b7640 -1 osd.1 0 waiting for initial osdmap 2026-03-24T02:13:48.780 INFO:tasks.ceph.cluster2.osd.1.vm07.stderr:2026-03-24T02:13:48.777+0000 7f2e15ea5640 -1 osd.1 0 waiting for initial osdmap 2026-03-24T02:13:48.783 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:13:48.777+0000 7f0862ceb640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-24T02:13:48.784 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:13:48.777+0000 7f77404c7640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-24T02:13:48.784 INFO:tasks.ceph.cluster2.osd.1.vm07.stderr:2026-03-24T02:13:48.781+0000 7f2e10cb3640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-24T02:13:52.674 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-03-24T02:13:52.865 INFO:teuthology.misc.health.vm07.stdout: 2026-03-24T02:13:52.866 INFO:teuthology.misc.health.vm07.stdout:{"epoch":10,"fsid":"91ca8768-ba68-432d-bc1f-efeea617a922","created":"2026-03-24T02:13:44.473145+0000","modified":"2026-03-24T02:13:52.226901+0000","last_up_change":"2026-03-24T02:13:49.762739+0000","last_in_change":"2026-03-24T02:13:45.559913+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-24T02:13:50.810447+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f02a1366-0e32-4132-9124-8b60215ba403","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6816","nonce":2403182922},{"type":"v1","addr":"192.168.123.107:6817","nonce":2403182922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6818","nonce":2403182922},{"type":"v1","addr":"192.168.123.107:6819","nonce":2403182922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6822","nonce":2403182922},{"type":"v1","addr":"192.168.123.107:6823","nonce":2403182922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6820","nonce":2403182922},{"type":"v1","addr":"192.168.123.107:6821","nonce":2403182922}]},"public_addr":"192.168.123.107:6817/2403182922","cluster_addr":"192.168.123.107:6819/2403182922","heartbeat_back_addr":"192.168.123.107:6823/2403182922","heartbeat_front_addr":"192.168.123.107:6821/2403182922","state":["exists","up"]},{"osd":1,"uuid":"c3c9032d-80f2-481b-b324-d91af15845ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6808","nonce":1372859641},{"type":"v1","addr":"192.168.123.107:6809","nonce":1372859641}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6810","nonce":1372859641},{"type":"v1","addr":"192.168.123.107:6811","nonce":1372859641}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6814","nonce":1372859641},{"type":"v1","addr":"192.168.123.107:6815","nonce":1372859641}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6812","nonce":1372859641},{"type":"v1","addr":"192.168.123.107:6813","nonce":1372859641}]},"public_addr":"192.168.123.107:6809/1372859641","cluster_addr":"192.168.123.107:6811/1372859641","heartbeat_back_addr":"192.168.123.107:6815/1372859641","heartbeat_front_addr":"192.168.123.107:6813/1372859641","state":["exists","up"]},{"osd":2,"uuid":"bca91acd-ce62-44d0-b940-2dcc1877d928","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6800","nonce":2006070968},{"type":"v1","addr":"192.168.123.107:6801","nonce":2006070968}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6802","nonce":2006070968},{"type":"v1","addr":"192.168.123.107:6803","nonce":2006070968}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6806","nonce":2006070968},{"type":"v1","addr":"192.168.123.107:6807","nonce":2006070968}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6804","nonce":2006070968},{"type":"v1","addr":"192.168.123.107:6805","nonce":2006070968}]},"public_addr":"192.168.123.107:6801/2006070968","cluster_addr":"192.168.123.107:6803/2006070968","heartbeat_back_addr":"192.168.123.107:6807/2006070968","heartbeat_front_addr":"192.168.123.107:6805/2006070968","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-24T02:13:47.664424+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-24T02:13:47.697934+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-24T02:13:47.730431+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-24T02:13:52.879 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-03-24T02:13:52.879 INFO:tasks.ceph:Creating RBD pool 2026-03-24T02:13:52.879 DEBUG:teuthology.orchestra.run.vm07:> sudo ceph --cluster cluster2 osd pool create rbd 8 2026-03-24T02:13:53.273 INFO:teuthology.orchestra.run.vm07.stderr:pool 'rbd' created 2026-03-24T02:13:53.296 DEBUG:teuthology.orchestra.run.vm07:> rbd --cluster cluster2 pool init rbd 2026-03-24T02:13:56.296 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2026-03-24T02:13:56.296 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 config log 1 --format=json 2026-03-24T02:13:56.296 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-03-24T02:13:56.478 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:56.493 INFO:teuthology.orchestra.run.vm07.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-03-24T02:13:56.493 INFO:tasks.ceph_manager:config epoch is 1 2026-03-24T02:13:56.493 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2026-03-24T02:13:56.493 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2026-03-24T02:13:56.493 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2026-03-24T02:13:56.680 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:56.695 INFO:teuthology.orchestra.run.vm07.stdout:{"epoch":5,"flags":0,"active_gid":4099,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6826","nonce":1748033232},{"type":"v1","addr":"192.168.123.106:6827","nonce":1748033232}]},"active_addr":"192.168.123.106:6827/1748033232","active_change":"2026-03-24T02:13:46.790926+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self certificates generated and signed by cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.5","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"how frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"192.168.123.106:0","nonce":3654478730}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"192.168.123.106:0","nonce":1802655479}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"192.168.123.106:0","nonce":1650315254}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"192.168.123.106:0","nonce":1967713412}]}]} 2026-03-24T02:13:56.696 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2026-03-24T02:13:56.696 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2026-03-24T02:13:56.696 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-03-24T02:13:56.877 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:56.877 INFO:teuthology.orchestra.run.vm07.stdout:{"epoch":14,"fsid":"91ca8768-ba68-432d-bc1f-efeea617a922","created":"2026-03-24T02:13:44.473145+0000","modified":"2026-03-24T02:13:56.284058+0000","last_up_change":"2026-03-24T02:13:49.762739+0000","last_in_change":"2026-03-24T02:13:45.559913+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-24T02:13:50.810447+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-03-24T02:13:53.059072+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f02a1366-0e32-4132-9124-8b60215ba403","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6816","nonce":2403182922},{"type":"v1","addr":"192.168.123.107:6817","nonce":2403182922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6818","nonce":2403182922},{"type":"v1","addr":"192.168.123.107:6819","nonce":2403182922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6822","nonce":2403182922},{"type":"v1","addr":"192.168.123.107:6823","nonce":2403182922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6820","nonce":2403182922},{"type":"v1","addr":"192.168.123.107:6821","nonce":2403182922}]},"public_addr":"192.168.123.107:6817/2403182922","cluster_addr":"192.168.123.107:6819/2403182922","heartbeat_back_addr":"192.168.123.107:6823/2403182922","heartbeat_front_addr":"192.168.123.107:6821/2403182922","state":["exists","up"]},{"osd":1,"uuid":"c3c9032d-80f2-481b-b324-d91af15845ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6808","nonce":1372859641},{"type":"v1","addr":"192.168.123.107:6809","nonce":1372859641}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6810","nonce":1372859641},{"type":"v1","addr":"192.168.123.107:6811","nonce":1372859641}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6814","nonce":1372859641},{"type":"v1","addr":"192.168.123.107:6815","nonce":1372859641}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6812","nonce":1372859641},{"type":"v1","addr":"192.168.123.107:6813","nonce":1372859641}]},"public_addr":"192.168.123.107:6809/1372859641","cluster_addr":"192.168.123.107:6811/1372859641","heartbeat_back_addr":"192.168.123.107:6815/1372859641","heartbeat_front_addr":"192.168.123.107:6813/1372859641","state":["exists","up"]},{"osd":2,"uuid":"bca91acd-ce62-44d0-b940-2dcc1877d928","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6800","nonce":2006070968},{"type":"v1","addr":"192.168.123.107:6801","nonce":2006070968}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6802","nonce":2006070968},{"type":"v1","addr":"192.168.123.107:6803","nonce":2006070968}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6806","nonce":2006070968},{"type":"v1","addr":"192.168.123.107:6807","nonce":2006070968}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6804","nonce":2006070968},{"type":"v1","addr":"192.168.123.107:6805","nonce":2006070968}]},"public_addr":"192.168.123.107:6801/2006070968","cluster_addr":"192.168.123.107:6803/2006070968","heartbeat_back_addr":"192.168.123.107:6807/2006070968","heartbeat_front_addr":"192.168.123.107:6805/2006070968","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-24T02:13:47.664424+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-24T02:13:47.697934+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-24T02:13:47.730431+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-24T02:13:56.893 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2026-03-24T02:13:56.893 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-03-24T02:13:57.059 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:13:57.059 INFO:teuthology.orchestra.run.vm07.stdout:{"epoch":14,"fsid":"91ca8768-ba68-432d-bc1f-efeea617a922","created":"2026-03-24T02:13:44.473145+0000","modified":"2026-03-24T02:13:56.284058+0000","last_up_change":"2026-03-24T02:13:49.762739+0000","last_in_change":"2026-03-24T02:13:45.559913+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-24T02:13:50.810447+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-03-24T02:13:53.059072+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f02a1366-0e32-4132-9124-8b60215ba403","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6816","nonce":2403182922},{"type":"v1","addr":"192.168.123.107:6817","nonce":2403182922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6818","nonce":2403182922},{"type":"v1","addr":"192.168.123.107:6819","nonce":2403182922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6822","nonce":2403182922},{"type":"v1","addr":"192.168.123.107:6823","nonce":2403182922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6820","nonce":2403182922},{"type":"v1","addr":"192.168.123.107:6821","nonce":2403182922}]},"public_addr":"192.168.123.107:6817/2403182922","cluster_addr":"192.168.123.107:6819/2403182922","heartbeat_back_addr":"192.168.123.107:6823/2403182922","heartbeat_front_addr":"192.168.123.107:6821/2403182922","state":["exists","up"]},{"osd":1,"uuid":"c3c9032d-80f2-481b-b324-d91af15845ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6808","nonce":1372859641},{"type":"v1","addr":"192.168.123.107:6809","nonce":1372859641}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6810","nonce":1372859641},{"type":"v1","addr":"192.168.123.107:6811","nonce":1372859641}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6814","nonce":1372859641},{"type":"v1","addr":"192.168.123.107:6815","nonce":1372859641}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6812","nonce":1372859641},{"type":"v1","addr":"192.168.123.107:6813","nonce":1372859641}]},"public_addr":"192.168.123.107:6809/1372859641","cluster_addr":"192.168.123.107:6811/1372859641","heartbeat_back_addr":"192.168.123.107:6815/1372859641","heartbeat_front_addr":"192.168.123.107:6813/1372859641","state":["exists","up"]},{"osd":2,"uuid":"bca91acd-ce62-44d0-b940-2dcc1877d928","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6800","nonce":2006070968},{"type":"v1","addr":"192.168.123.107:6801","nonce":2006070968}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6802","nonce":2006070968},{"type":"v1","addr":"192.168.123.107:6803","nonce":2006070968}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6806","nonce":2006070968},{"type":"v1","addr":"192.168.123.107:6807","nonce":2006070968}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6804","nonce":2006070968},{"type":"v1","addr":"192.168.123.107:6805","nonce":2006070968}]},"public_addr":"192.168.123.107:6801/2006070968","cluster_addr":"192.168.123.107:6803/2006070968","heartbeat_back_addr":"192.168.123.107:6807/2006070968","heartbeat_front_addr":"192.168.123.107:6805/2006070968","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-24T02:13:47.664424+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-24T02:13:47.697934+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-24T02:13:47.730431+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-24T02:13:57.076 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2026-03-24T02:13:57.076 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2026-03-24T02:13:57.076 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2026-03-24T02:13:57.189 INFO:teuthology.orchestra.run.vm07.stdout:34359738371 2026-03-24T02:13:57.190 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-03-24T02:13:57.190 INFO:teuthology.orchestra.run.vm07.stdout:34359738371 2026-03-24T02:13:57.190 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-03-24T02:13:57.196 INFO:teuthology.orchestra.run.vm07.stdout:34359738372 2026-03-24T02:13:57.196 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-03-24T02:13:57.388 INFO:teuthology.orchestra.run.vm07.stdout:34359738370 2026-03-24T02:13:57.402 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.1 2026-03-24T02:13:57.427 INFO:teuthology.orchestra.run.vm07.stdout:34359738370 2026-03-24T02:13:57.442 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.2 2026-03-24T02:13:57.442 INFO:teuthology.orchestra.run.vm07.stdout:34359738371 2026-03-24T02:13:57.457 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.0 2026-03-24T02:13:58.403 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-03-24T02:13:58.443 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-03-24T02:13:58.457 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-03-24T02:13:58.581 INFO:teuthology.orchestra.run.vm07.stdout:34359738370 2026-03-24T02:13:58.596 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.1 2026-03-24T02:13:58.630 INFO:teuthology.orchestra.run.vm07.stdout:34359738370 2026-03-24T02:13:58.646 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.2 2026-03-24T02:13:58.672 INFO:teuthology.orchestra.run.vm07.stdout:34359738371 2026-03-24T02:13:58.688 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.0 2026-03-24T02:13:59.597 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-03-24T02:13:59.647 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-03-24T02:13:59.689 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-03-24T02:13:59.783 INFO:teuthology.orchestra.run.vm07.stdout:34359738371 2026-03-24T02:13:59.799 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738371 for osd.1 2026-03-24T02:13:59.799 DEBUG:teuthology.parallel:result is None 2026-03-24T02:13:59.837 INFO:teuthology.orchestra.run.vm07.stdout:34359738371 2026-03-24T02:13:59.852 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738371 for osd.2 2026-03-24T02:13:59.852 DEBUG:teuthology.parallel:result is None 2026-03-24T02:13:59.874 INFO:teuthology.orchestra.run.vm07.stdout:34359738372 2026-03-24T02:13:59.889 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738372 for osd.0 2026-03-24T02:13:59.889 DEBUG:teuthology.parallel:result is None 2026-03-24T02:13:59.889 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-03-24T02:13:59.889 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-03-24T02:14:00.111 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:14:00.112 INFO:teuthology.orchestra.run.vm07.stderr:dumped all 2026-03-24T02:14:00.126 INFO:teuthology.orchestra.run.vm07.stdout:{"pg_ready":true,"pg_map":{"version":18,"stamp":"2026-03-24T02:13:59.468930+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":459299,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":59,"num_write_kb":586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35,"ondisk_log_size":35,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":14,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81800,"kb_used_data":1264,"kb_used_omap":22,"kb_used_meta":80425,"kb_avail":283033720,"statfs":{"total":289910292480,"available":289826529280,"internally_reserved":0,"allocated":1294336,"data_stored":1021524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":23062,"internal_metadata":82355690},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":11,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.182682"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.290698+0000","last_change":"2026-03-24T02:13:56.291116+0000","last_active":"2026-03-24T02:13:56.290698+0000","last_peered":"2026-03-24T02:13:56.290698+0000","last_clean":"2026-03-24T02:13:56.290698+0000","last_became_active":"2026-03-24T02:13:54.285179+0000","last_became_peered":"2026-03-24T02:13:54.285179+0000","last_unstale":"2026-03-24T02:13:56.290698+0000","last_undegraded":"2026-03-24T02:13:56.290698+0000","last_fullsized":"2026-03-24T02:13:56.290698+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:40:35.710968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052393000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.290782+0000","last_change":"2026-03-24T02:13:56.291113+0000","last_active":"2026-03-24T02:13:56.290782+0000","last_peered":"2026-03-24T02:13:56.290782+0000","last_clean":"2026-03-24T02:13:56.290782+0000","last_became_active":"2026-03-24T02:13:54.286185+0000","last_became_peered":"2026-03-24T02:13:54.286185+0000","last_unstale":"2026-03-24T02:13:56.290782+0000","last_undegraded":"2026-03-24T02:13:56.290782+0000","last_fullsized":"2026-03-24T02:13:56.290782+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:50:07.713703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041187999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.291148+0000","last_change":"2026-03-24T02:13:56.291271+0000","last_active":"2026-03-24T02:13:56.291148+0000","last_peered":"2026-03-24T02:13:56.291148+0000","last_clean":"2026-03-24T02:13:56.291148+0000","last_became_active":"2026-03-24T02:13:54.284043+0000","last_became_peered":"2026-03-24T02:13:54.284043+0000","last_unstale":"2026-03-24T02:13:56.291148+0000","last_undegraded":"2026-03-24T02:13:56.291148+0000","last_fullsized":"2026-03-24T02:13:56.291148+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:42:40.008023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045964999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.291210+0000","last_change":"2026-03-24T02:13:56.291308+0000","last_active":"2026-03-24T02:13:56.291210+0000","last_peered":"2026-03-24T02:13:56.291210+0000","last_clean":"2026-03-24T02:13:56.291210+0000","last_became_active":"2026-03-24T02:13:54.283892+0000","last_became_peered":"2026-03-24T02:13:54.283892+0000","last_unstale":"2026-03-24T02:13:56.291210+0000","last_undegraded":"2026-03-24T02:13:56.291210+0000","last_fullsized":"2026-03-24T02:13:56.291210+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T08:17:26.341514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039982800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"14'2","reported_seq":22,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.288111+0000","last_change":"2026-03-24T02:13:56.288111+0000","last_active":"2026-03-24T02:13:56.288111+0000","last_peered":"2026-03-24T02:13:56.288111+0000","last_clean":"2026-03-24T02:13:56.288111+0000","last_became_active":"2026-03-24T02:13:54.283188+0000","last_became_peered":"2026-03-24T02:13:54.283188+0000","last_unstale":"2026-03-24T02:13:56.288111+0000","last_undegraded":"2026-03-24T02:13:56.288111+0000","last_fullsized":"2026-03-24T02:13:56.288111+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:58:21.144364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031652200000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.736113+0000","last_change":"2026-03-24T02:13:56.736302+0000","last_active":"2026-03-24T02:13:56.736113+0000","last_peered":"2026-03-24T02:13:56.736113+0000","last_clean":"2026-03-24T02:13:56.736113+0000","last_became_active":"2026-03-24T02:13:54.284823+0000","last_became_peered":"2026-03-24T02:13:54.284823+0000","last_unstale":"2026-03-24T02:13:56.736113+0000","last_undegraded":"2026-03-24T02:13:56.736113+0000","last_fullsized":"2026-03-24T02:13:56.736113+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T12:50:46.136937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000322784,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.736171+0000","last_change":"2026-03-24T02:13:56.736294+0000","last_active":"2026-03-24T02:13:56.736171+0000","last_peered":"2026-03-24T02:13:56.736171+0000","last_clean":"2026-03-24T02:13:56.736171+0000","last_became_active":"2026-03-24T02:13:54.284987+0000","last_became_peered":"2026-03-24T02:13:54.284987+0000","last_unstale":"2026-03-24T02:13:56.736171+0000","last_undegraded":"2026-03-24T02:13:56.736171+0000","last_fullsized":"2026-03-24T02:13:56.736171+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:52:45.132362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025170100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"12'1","reported_seq":21,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.291340+0000","last_change":"2026-03-24T02:13:56.291437+0000","last_active":"2026-03-24T02:13:56.291340+0000","last_peered":"2026-03-24T02:13:56.291340+0000","last_clean":"2026-03-24T02:13:56.291340+0000","last_became_active":"2026-03-24T02:13:54.283277+0000","last_became_peered":"2026-03-24T02:13:54.283277+0000","last_unstale":"2026-03-24T02:13:56.291340+0000","last_undegraded":"2026-03-24T02:13:56.291340+0000","last_fullsized":"2026-03-24T02:13:56.291340+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T10:25:36.882503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039834499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'32","reported_seq":63,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.290991+0000","last_change":"2026-03-24T02:13:52.693536+0000","last_active":"2026-03-24T02:13:56.290991+0000","last_peered":"2026-03-24T02:13:56.290991+0000","last_clean":"2026-03-24T02:13:56.290991+0000","last_became_active":"2026-03-24T02:13:52.693283+0000","last_became_peered":"2026-03-24T02:13:52.693283+0000","last_unstale":"2026-03-24T02:13:56.290991+0000","last_undegraded":"2026-03-24T02:13:56.290991+0000","last_fullsized":"2026-03-24T02:13:56.290991+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:51.152553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:51.152553+0000","last_clean_scrub_stamp":"2026-03-24T02:13:51.152553+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:26:45.574238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":925696,"data_stored":918560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32,"ondisk_log_size":32,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738371,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":26968,"kb_used_data":112,"kb_used_omap":6,"kb_used_meta":26809,"kb_avail":94344872,"statfs":{"total":96636764160,"available":96609148928,"internally_reserved":0,"allocated":114688,"data_stored":31938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6821,"internal_metadata":27452763},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738371,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27416,"kb_used_data":576,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":94344424,"statfs":{"total":96636764160,"available":96608690176,"internally_reserved":0,"allocated":589824,"data_stored":494793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8121,"internal_metadata":27451463},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27416,"kb_used_data":576,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":94344424,"statfs":{"total":96636764160,"available":96608690176,"internally_reserved":0,"allocated":589824,"data_stored":494793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8120,"internal_metadata":27451464},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-24T02:14:00.126 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-03-24T02:14:00.322 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:14:00.323 INFO:teuthology.orchestra.run.vm07.stderr:dumped all 2026-03-24T02:14:00.337 INFO:teuthology.orchestra.run.vm07.stdout:{"pg_ready":true,"pg_map":{"version":18,"stamp":"2026-03-24T02:13:59.468930+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":459299,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":59,"num_write_kb":586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35,"ondisk_log_size":35,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":14,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81800,"kb_used_data":1264,"kb_used_omap":22,"kb_used_meta":80425,"kb_avail":283033720,"statfs":{"total":289910292480,"available":289826529280,"internally_reserved":0,"allocated":1294336,"data_stored":1021524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":23062,"internal_metadata":82355690},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":11,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.182682"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.290698+0000","last_change":"2026-03-24T02:13:56.291116+0000","last_active":"2026-03-24T02:13:56.290698+0000","last_peered":"2026-03-24T02:13:56.290698+0000","last_clean":"2026-03-24T02:13:56.290698+0000","last_became_active":"2026-03-24T02:13:54.285179+0000","last_became_peered":"2026-03-24T02:13:54.285179+0000","last_unstale":"2026-03-24T02:13:56.290698+0000","last_undegraded":"2026-03-24T02:13:56.290698+0000","last_fullsized":"2026-03-24T02:13:56.290698+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:40:35.710968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052393000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.290782+0000","last_change":"2026-03-24T02:13:56.291113+0000","last_active":"2026-03-24T02:13:56.290782+0000","last_peered":"2026-03-24T02:13:56.290782+0000","last_clean":"2026-03-24T02:13:56.290782+0000","last_became_active":"2026-03-24T02:13:54.286185+0000","last_became_peered":"2026-03-24T02:13:54.286185+0000","last_unstale":"2026-03-24T02:13:56.290782+0000","last_undegraded":"2026-03-24T02:13:56.290782+0000","last_fullsized":"2026-03-24T02:13:56.290782+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:50:07.713703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041187999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.291148+0000","last_change":"2026-03-24T02:13:56.291271+0000","last_active":"2026-03-24T02:13:56.291148+0000","last_peered":"2026-03-24T02:13:56.291148+0000","last_clean":"2026-03-24T02:13:56.291148+0000","last_became_active":"2026-03-24T02:13:54.284043+0000","last_became_peered":"2026-03-24T02:13:54.284043+0000","last_unstale":"2026-03-24T02:13:56.291148+0000","last_undegraded":"2026-03-24T02:13:56.291148+0000","last_fullsized":"2026-03-24T02:13:56.291148+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:42:40.008023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045964999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.291210+0000","last_change":"2026-03-24T02:13:56.291308+0000","last_active":"2026-03-24T02:13:56.291210+0000","last_peered":"2026-03-24T02:13:56.291210+0000","last_clean":"2026-03-24T02:13:56.291210+0000","last_became_active":"2026-03-24T02:13:54.283892+0000","last_became_peered":"2026-03-24T02:13:54.283892+0000","last_unstale":"2026-03-24T02:13:56.291210+0000","last_undegraded":"2026-03-24T02:13:56.291210+0000","last_fullsized":"2026-03-24T02:13:56.291210+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T08:17:26.341514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039982800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"14'2","reported_seq":22,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.288111+0000","last_change":"2026-03-24T02:13:56.288111+0000","last_active":"2026-03-24T02:13:56.288111+0000","last_peered":"2026-03-24T02:13:56.288111+0000","last_clean":"2026-03-24T02:13:56.288111+0000","last_became_active":"2026-03-24T02:13:54.283188+0000","last_became_peered":"2026-03-24T02:13:54.283188+0000","last_unstale":"2026-03-24T02:13:56.288111+0000","last_undegraded":"2026-03-24T02:13:56.288111+0000","last_fullsized":"2026-03-24T02:13:56.288111+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:58:21.144364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031652200000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.736113+0000","last_change":"2026-03-24T02:13:56.736302+0000","last_active":"2026-03-24T02:13:56.736113+0000","last_peered":"2026-03-24T02:13:56.736113+0000","last_clean":"2026-03-24T02:13:56.736113+0000","last_became_active":"2026-03-24T02:13:54.284823+0000","last_became_peered":"2026-03-24T02:13:54.284823+0000","last_unstale":"2026-03-24T02:13:56.736113+0000","last_undegraded":"2026-03-24T02:13:56.736113+0000","last_fullsized":"2026-03-24T02:13:56.736113+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T12:50:46.136937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000322784,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.736171+0000","last_change":"2026-03-24T02:13:56.736294+0000","last_active":"2026-03-24T02:13:56.736171+0000","last_peered":"2026-03-24T02:13:56.736171+0000","last_clean":"2026-03-24T02:13:56.736171+0000","last_became_active":"2026-03-24T02:13:54.284987+0000","last_became_peered":"2026-03-24T02:13:54.284987+0000","last_unstale":"2026-03-24T02:13:56.736171+0000","last_undegraded":"2026-03-24T02:13:56.736171+0000","last_fullsized":"2026-03-24T02:13:56.736171+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:52:45.132362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025170100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"12'1","reported_seq":21,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.291340+0000","last_change":"2026-03-24T02:13:56.291437+0000","last_active":"2026-03-24T02:13:56.291340+0000","last_peered":"2026-03-24T02:13:56.291340+0000","last_clean":"2026-03-24T02:13:56.291340+0000","last_became_active":"2026-03-24T02:13:54.283277+0000","last_became_peered":"2026-03-24T02:13:54.283277+0000","last_unstale":"2026-03-24T02:13:56.291340+0000","last_undegraded":"2026-03-24T02:13:56.291340+0000","last_fullsized":"2026-03-24T02:13:56.291340+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T10:25:36.882503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039834499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'32","reported_seq":63,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-03-24T02:13:56.290991+0000","last_change":"2026-03-24T02:13:52.693536+0000","last_active":"2026-03-24T02:13:56.290991+0000","last_peered":"2026-03-24T02:13:56.290991+0000","last_clean":"2026-03-24T02:13:56.290991+0000","last_became_active":"2026-03-24T02:13:52.693283+0000","last_became_peered":"2026-03-24T02:13:52.693283+0000","last_unstale":"2026-03-24T02:13:56.290991+0000","last_undegraded":"2026-03-24T02:13:56.290991+0000","last_fullsized":"2026-03-24T02:13:56.290991+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:51.152553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:51.152553+0000","last_clean_scrub_stamp":"2026-03-24T02:13:51.152553+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:26:45.574238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":925696,"data_stored":918560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32,"ondisk_log_size":32,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738371,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":26968,"kb_used_data":112,"kb_used_omap":6,"kb_used_meta":26809,"kb_avail":94344872,"statfs":{"total":96636764160,"available":96609148928,"internally_reserved":0,"allocated":114688,"data_stored":31938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6821,"internal_metadata":27452763},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738371,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27416,"kb_used_data":576,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":94344424,"statfs":{"total":96636764160,"available":96608690176,"internally_reserved":0,"allocated":589824,"data_stored":494793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8121,"internal_metadata":27451463},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27416,"kb_used_data":576,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":94344424,"statfs":{"total":96636764160,"available":96608690176,"internally_reserved":0,"allocated":589824,"data_stored":494793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8120,"internal_metadata":27451464},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-24T02:14:00.337 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-03-24T02:14:00.337 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2026-03-24T02:14:00.337 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2026-03-24T02:14:00.337 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2026-03-24T02:14:00.529 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:14:00.529 INFO:teuthology.orchestra.run.vm07.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-03-24T02:14:00.544 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2026-03-24T02:14:00.544 INFO:teuthology.run_tasks:Running task exec... 2026-03-24T02:14:00.547 INFO:teuthology.task.exec:Executing custom commands... 2026-03-24T02:14:00.547 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@vm07.local 2026-03-24T02:14:00.547 DEBUG:teuthology.orchestra.run.vm07:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-03-24T02:14:00.807 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror] 2026-03-24T02:14:00.807 INFO:teuthology.orchestra.run.vm07.stdout: key = AQA688FpKYaAERAAOxraWo2temFqR9WiveBVRA== 2026-03-24T02:14:00.807 INFO:teuthology.orchestra.run.vm07.stdout: caps mon = "profile rbd-mirror-peer" 2026-03-24T02:14:00.807 INFO:teuthology.orchestra.run.vm07.stdout: caps osd = "profile rbd" 2026-03-24T02:14:00.807 INFO:teuthology.orchestra.run.vm07.stderr:updated caps for client.mirror 2026-03-24T02:14:00.823 DEBUG:teuthology.orchestra.run.vm07:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-24T02:14:01.046 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.0] 2026-03-24T02:14:01.047 INFO:teuthology.orchestra.run.vm07.stdout: key = AQA688FpzpeaFRAArS5MDmAi5cYJl2sR+6MhJg== 2026-03-24T02:14:01.047 INFO:teuthology.orchestra.run.vm07.stdout: caps mon = "profile rbd-mirror" 2026-03-24T02:14:01.047 INFO:teuthology.orchestra.run.vm07.stdout: caps osd = "profile rbd" 2026-03-24T02:14:01.047 INFO:teuthology.orchestra.run.vm07.stderr:updated caps for client.mirror.0 2026-03-24T02:14:01.064 DEBUG:teuthology.orchestra.run.vm07:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-24T02:14:01.388 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.1] 2026-03-24T02:14:01.388 INFO:teuthology.orchestra.run.vm07.stdout: key = AQA688Fpekm7GRAAMhIopHgVDYLYvlQ3uSdeYA== 2026-03-24T02:14:01.388 INFO:teuthology.orchestra.run.vm07.stdout: caps mon = "profile rbd-mirror" 2026-03-24T02:14:01.388 INFO:teuthology.orchestra.run.vm07.stdout: caps osd = "profile rbd" 2026-03-24T02:14:01.388 INFO:teuthology.orchestra.run.vm07.stderr:updated caps for client.mirror.1 2026-03-24T02:14:01.403 DEBUG:teuthology.orchestra.run.vm07:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-24T02:14:01.614 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.2] 2026-03-24T02:14:01.614 INFO:teuthology.orchestra.run.vm07.stdout: key = AQA688FpXR7FHRAA4grqZRb8dCATG6ua++0Xwg== 2026-03-24T02:14:01.614 INFO:teuthology.orchestra.run.vm07.stdout: caps mon = "profile rbd-mirror" 2026-03-24T02:14:01.614 INFO:teuthology.orchestra.run.vm07.stdout: caps osd = "profile rbd" 2026-03-24T02:14:01.614 INFO:teuthology.orchestra.run.vm07.stderr:updated caps for client.mirror.2 2026-03-24T02:14:01.631 DEBUG:teuthology.orchestra.run.vm07:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-24T02:14:01.846 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.3] 2026-03-24T02:14:01.846 INFO:teuthology.orchestra.run.vm07.stdout: key = AQA688FpbSzMIRAAvTDMMJH70T5hd4qEd4B4gw== 2026-03-24T02:14:01.846 INFO:teuthology.orchestra.run.vm07.stdout: caps mon = "profile rbd-mirror" 2026-03-24T02:14:01.846 INFO:teuthology.orchestra.run.vm07.stdout: caps osd = "profile rbd" 2026-03-24T02:14:01.846 INFO:teuthology.orchestra.run.vm07.stderr:updated caps for client.mirror.3 2026-03-24T02:14:01.861 DEBUG:teuthology.orchestra.run.vm07:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-03-24T02:14:02.080 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror] 2026-03-24T02:14:02.080 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBS88FpX+HrABAAeI6XlW8Y+zinDD9KOq4Blg== 2026-03-24T02:14:02.080 INFO:teuthology.orchestra.run.vm07.stdout: caps mon = "profile rbd-mirror-peer" 2026-03-24T02:14:02.080 INFO:teuthology.orchestra.run.vm07.stdout: caps osd = "profile rbd" 2026-03-24T02:14:02.080 INFO:teuthology.orchestra.run.vm07.stderr:updated caps for client.mirror 2026-03-24T02:14:02.096 DEBUG:teuthology.orchestra.run.vm07:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-24T02:14:02.320 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.0] 2026-03-24T02:14:02.320 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBS88FpFqoEBRAAiWzcM4LV0Ow+JbKs9KY1VA== 2026-03-24T02:14:02.320 INFO:teuthology.orchestra.run.vm07.stdout: caps mon = "profile rbd-mirror" 2026-03-24T02:14:02.320 INFO:teuthology.orchestra.run.vm07.stdout: caps osd = "profile rbd" 2026-03-24T02:14:02.320 INFO:teuthology.orchestra.run.vm07.stderr:updated caps for client.mirror.0 2026-03-24T02:14:02.336 DEBUG:teuthology.orchestra.run.vm07:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-24T02:14:02.552 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.1] 2026-03-24T02:14:02.552 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBS88FpyLoNCRAAqlwDI+7jCTFHZe8bnBig3A== 2026-03-24T02:14:02.552 INFO:teuthology.orchestra.run.vm07.stdout: caps mon = "profile rbd-mirror" 2026-03-24T02:14:02.552 INFO:teuthology.orchestra.run.vm07.stdout: caps osd = "profile rbd" 2026-03-24T02:14:02.552 INFO:teuthology.orchestra.run.vm07.stderr:updated caps for client.mirror.1 2026-03-24T02:14:02.567 DEBUG:teuthology.orchestra.run.vm07:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-24T02:14:02.786 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.2] 2026-03-24T02:14:02.786 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBS88FppxccDRAA6dYCZqy1MhT+ZmqRrR5hYw== 2026-03-24T02:14:02.786 INFO:teuthology.orchestra.run.vm07.stdout: caps mon = "profile rbd-mirror" 2026-03-24T02:14:02.786 INFO:teuthology.orchestra.run.vm07.stdout: caps osd = "profile rbd" 2026-03-24T02:14:02.786 INFO:teuthology.orchestra.run.vm07.stderr:updated caps for client.mirror.2 2026-03-24T02:14:02.802 DEBUG:teuthology.orchestra.run.vm07:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-24T02:14:03.022 INFO:teuthology.orchestra.run.vm07.stdout:[client.mirror.3] 2026-03-24T02:14:03.022 INFO:teuthology.orchestra.run.vm07.stdout: key = AQBS88Fp4FUwERAAW4sC/UfOh+zPZQzJd7hOOQ== 2026-03-24T02:14:03.023 INFO:teuthology.orchestra.run.vm07.stdout: caps mon = "profile rbd-mirror" 2026-03-24T02:14:03.023 INFO:teuthology.orchestra.run.vm07.stdout: caps osd = "profile rbd" 2026-03-24T02:14:03.023 INFO:teuthology.orchestra.run.vm07.stderr:updated caps for client.mirror.3 2026-03-24T02:14:03.040 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-03-24T02:14:03.043 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-03-24T02:14:03.043 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-03-24T02:14:03.044 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-03-24T02:14:03.044 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-03-24T02:14:03.046 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-03-24T02:14:03.046 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-03-24T02:14:03.047 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-03-24T02:14:03.047 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-03-24T02:14:03.049 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-03-24T02:14:03.049 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-03-24T02:14:03.050 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-03-24T02:14:03.050 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-03-24T02:14:03.052 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-03-24T02:14:03.052 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-03-24T02:14:03.053 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-03-24T02:14:03.053 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-03-24T02:14:03.056 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-03-24T02:14:03.056 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-03-24T02:14:03.059 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-03-24T02:14:03.059 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-03-24T02:14:03.062 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-03-24T02:14:03.062 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-03-24T02:14:03.062 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-03-24T02:14:03.062 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-03-24T02:14:03.088 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-03-24T02:14:03.088 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-03-24T02:14:03.090 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-03-24T02:14:03.090 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-03-24T02:14:03.093 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-03-24T02:14:03.093 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-03-24T02:14:03.094 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-03-24T02:14:03.094 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2026-03-24T02:14:03.097 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1774318443 2026-03-24T02:14:03.097 DEBUG:tasks.rbd_mirror_thrash:Yielding 2026-03-24T02:14:03.097 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2026-03-24T02:14:03.100 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1774318443 2026-03-24T02:14:03.100 DEBUG:tasks.rbd_mirror_thrash:Yielding 2026-03-24T02:14:03.100 INFO:teuthology.run_tasks:Running task workunit... 2026-03-24T02:14:03.105 INFO:tasks.workunit:Pulling workunits from ref 0392f78529848ec72469e8e431875cb98d3a5fb4 2026-03-24T02:14:03.105 INFO:tasks.workunit:Making a separate scratch dir for every client... 2026-03-24T02:14:03.105 DEBUG:teuthology.orchestra.run.vm07:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-03-24T02:14:03.105 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster1 2026-03-24T02:14:03.105 INFO:tasks.rbd_mirror_thrash:waiting for 75.07704574444794 secs before thrashing 2026-03-24T02:14:03.105 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster2 2026-03-24T02:14:03.105 INFO:tasks.rbd_mirror_thrash:waiting for 88.47801416762002 secs before thrashing 2026-03-24T02:14:03.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-24T02:14:03.120 INFO:teuthology.orchestra.run.vm07.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2026-03-24T02:14:03.121 DEBUG:teuthology.orchestra.run.vm07:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-03-24T02:14:03.168 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-03-24T02:14:03.168 DEBUG:teuthology.orchestra.run.vm07:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2026-03-24T02:14:03.214 INFO:tasks.workunit:timeout=6h 2026-03-24T02:14:03.214 INFO:tasks.workunit:cleanup=True 2026-03-24T02:14:03.215 DEBUG:teuthology.orchestra.run.vm07:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://github.com/kshtsk/ceph.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout 0392f78529848ec72469e8e431875cb98d3a5fb4 2026-03-24T02:14:03.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Note: switching to '0392f78529848ec72469e8e431875cb98d3a5fb4'. 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:changes and commit them, and you can discard any commits you make in this 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:state without impacting any branches by switching back to a branch. 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:If you want to create a new branch to retain commits you create, you may 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:do so (now or later) by using -c with the switch command. Example: 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: git switch -c 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Or undo this operation with: 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: git switch - 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: 2026-03-24T02:14:49.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:HEAD is now at 0392f785298 qa/tasks/keystone: restart mariadb for rocky and alma linux too 2026-03-24T02:14:49.893 DEBUG:teuthology.orchestra.run.vm07:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2026-03-24T02:14:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2026-03-24T02:14:49.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-03-24T02:14:49.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2026-03-24T02:14:49.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2026-03-24T02:14:50.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2026-03-24T02:14:50.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-03-24T02:14:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-03-24T02:14:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2026-03-24T02:14:50.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-03-24T02:14:50.087 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-24T02:14:50.087 DEBUG:teuthology.orchestra.run.vm07:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2026-03-24T02:14:50.132 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror_stress.sh on cluster1.client.mirror... 2026-03-24T02:14:50.132 INFO:tasks.workunit:Running workunit rbd/rbd_mirror_stress.sh... 2026-03-24T02:14:50.132 DEBUG:teuthology.orchestra.run.vm07:workunit test rbd/rbd_mirror_stress.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0392f78529848ec72469e8e431875cb98d3a5fb4 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' MIRROR_POOL_MODE=pool RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=journal RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-03-24T02:14:50.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ IMAGE_COUNT=50 2026-03-24T02:14:50.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ export LOCKDEP=0 2026-03-24T02:14:50.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ LOCKDEP=0 2026-03-24T02:14:50.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-03-24T02:14:50.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2026-03-24T02:14:50.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ RBD_MIRROR_INSTANCES=4 2026-03-24T02:14:50.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CLUSTER1=cluster1 2026-03-24T02:14:50.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CLUSTER2=cluster2 2026-03-24T02:14:50.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ PEER_CLUSTER_SUFFIX= 2026-03-24T02:14:50.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ POOL=mirror 2026-03-24T02:14:50.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ PARENT_POOL=mirror_parent 2026-03-24T02:14:50.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ NS1=ns1 2026-03-24T02:14:50.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ NS2=ns2 2026-03-24T02:14:50.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ TEMPDIR= 2026-03-24T02:14:50.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ID=mirror 2026-03-24T02:14:50.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock,journaling 2026-03-24T02:14:50.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2026-03-24T02:14:50.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ RBD_MIRROR_MODE=journal 2026-03-24T02:14:50.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ MIRROR_POOL_MODE=pool 2026-03-24T02:14:50.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' journal = snapshot ']' 2026-03-24T02:14:50.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ export 'CEPH_ARGS=--id mirror' 2026-03-24T02:14:50.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS='--id mirror' 2026-03-24T02:14:50.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ LAST_MIRROR_INSTANCE=3 2026-03-24T02:14:50.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-03-24T02:14:50.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2026-03-24T02:14:50.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2026-03-24T02:14:50.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_BIN=. 2026-03-24T02:14:50.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_SRC=. 2026-03-24T02:14:50.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' -e CMakeCache.txt ']' 2026-03-24T02:14:50.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 -gt 0 ']' 2026-03-24T02:14:50.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ setup 2026-03-24T02:14:50.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local c 2026-03-24T02:14:50.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ trap 'cleanup $?' INT TERM EXIT 2026-03-24T02:14:50.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ setup_tempdir 2026-03-24T02:14:50.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:14:50.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ mktemp -d 2026-03-24T02:14:50.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ TEMPDIR=/tmp/tmp.l3GEA596QR 2026-03-24T02:14:50.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -z 1 ']' 2026-03-24T02:14:50.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ setup_pools cluster1 cluster2 2026-03-24T02:14:50.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:14:50.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_cluster=cluster2 2026-03-24T02:14:50.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mon_map_file 2026-03-24T02:14:50.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mon_addr 2026-03-24T02:14:50.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local admin_key_file 2026-03-24T02:14:50.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local uuid 2026-03-24T02:14:50.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ CEPH_ARGS= 2026-03-24T02:14:50.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2026-03-24T02:14:50.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:pool 'mirror' already exists 2026-03-24T02:14:50.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ CEPH_ARGS= 2026-03-24T02:14:50.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2026-03-24T02:14:51.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:pool 'mirror_parent' already exists 2026-03-24T02:14:51.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ CEPH_ARGS= 2026-03-24T02:14:51.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 pool init mirror 2026-03-24T02:14:54.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ CEPH_ARGS= 2026-03-24T02:14:54.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2026-03-24T02:14:58.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:14:58.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ CEPH_ARGS= 2026-03-24T02:14:58.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1 mirror pool 2026-03-24T02:14:58.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2026-03-24T02:14:58.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2026-03-24T02:14:58.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2026-03-24T02:14:58.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 namespace create mirror_parent/ns1 2026-03-24T02:14:58.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 pool 2026-03-24T02:14:58.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2026-03-24T02:14:58.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent/ns1 pool 2026-03-24T02:14:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -z ']' 2026-03-24T02:14:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -z ']' 2026-03-24T02:14:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ peer_add cluster1 mirror cluster2 2026-03-24T02:14:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:14:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:14:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:14:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:14:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local client_cluster=cluster2 2026-03-24T02:14:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:14:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_cluster=cluster2 2026-03-24T02:14:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local uuid_var_name 2026-03-24T02:14:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:14:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local error_code 2026-03-24T02:14:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local peer_uuid 2026-03-24T02:14:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 16 32 2026-03-24T02:14:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:14:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror cluster2 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ peer_uuid=dddca223-77d9-43ad-a869-0470d458d5b2 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=0 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 -eq 17 ']' 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 0 -eq 0 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ peer_add cluster1 mirror_parent cluster2 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror_parent 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local client_cluster=cluster2 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_cluster=cluster2 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local uuid_var_name 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local error_code 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local peer_uuid 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 16 32 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:14:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent cluster2 2026-03-24T02:14:58.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ peer_uuid=b711d228-8fb4-4ed9-91b8-89c31ba9f34d 2026-03-24T02:14:58.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=0 2026-03-24T02:14:58.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:14:58.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 -eq 17 ']' 2026-03-24T02:14:58.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 0 -eq 0 2026-03-24T02:14:58.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:14:58.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:14:58.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ setup_pools cluster2 cluster1 2026-03-24T02:14:58.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:14:58.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_cluster=cluster1 2026-03-24T02:14:58.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mon_map_file 2026-03-24T02:14:58.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mon_addr 2026-03-24T02:14:58.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local admin_key_file 2026-03-24T02:14:58.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local uuid 2026-03-24T02:14:58.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ CEPH_ARGS= 2026-03-24T02:14:58.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2026-03-24T02:14:59.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:pool 'mirror' already exists 2026-03-24T02:14:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ CEPH_ARGS= 2026-03-24T02:14:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2026-03-24T02:15:00.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:pool 'mirror_parent' already exists 2026-03-24T02:15:00.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ CEPH_ARGS= 2026-03-24T02:15:00.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 pool init mirror 2026-03-24T02:15:03.149 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:15:03.145+0000 7fed9ae72980 -1 rbd::mirror::PoolReplayer: 0x55623f194c00 init: failed to initialize remote pool poller: (2) No such file or directory 2026-03-24T02:15:03.153 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:15:03.149+0000 7f504c974980 -1 rbd::mirror::PoolReplayer: 0x55ee0b198c00 init: failed to initialize remote pool poller: (2) No such file or directory 2026-03-24T02:15:03.160 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:15:03.153+0000 7fe8af94d980 -1 rbd::mirror::PoolReplayer: 0x55ea25fc4c00 init: failed to initialize remote pool poller: (2) No such file or directory 2026-03-24T02:15:03.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:15:03.157+0000 7f414e356980 -1 rbd::mirror::PoolReplayer: 0x55764c7bcc00 init: failed to initialize remote pool poller: (2) No such file or directory 2026-03-24T02:15:03.216 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:15:03.213+0000 7fe8af94d980 -1 rbd::mirror::PoolReplayer: 0x55ea26d47800 init: failed to initialize remote pool poller: (2) No such file or directory 2026-03-24T02:15:03.216 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:15:03.213+0000 7f414e356980 -1 rbd::mirror::PoolReplayer: 0x55764d55b800 init: failed to initialize remote pool poller: (2) No such file or directory 2026-03-24T02:15:03.385 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:15:03.381+0000 7fed9ae72980 -1 rbd::mirror::PoolReplayer: 0x55623f2d7800 init: failed to initialize remote pool poller: (2) No such file or directory 2026-03-24T02:15:03.398 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:15:03.393+0000 7f504c974980 -1 rbd::mirror::PoolReplayer: 0x55ee0bf17500 init: failed to initialize remote pool poller: (2) No such file or directory 2026-03-24T02:15:03.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ CEPH_ARGS= 2026-03-24T02:15:03.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2026-03-24T02:15:06.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:15:06.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ CEPH_ARGS= 2026-03-24T02:15:06.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2 mirror pool 2026-03-24T02:15:06.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2026-03-24T02:15:06.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2026-03-24T02:15:07.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2026-03-24T02:15:07.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 namespace create mirror_parent/ns1 2026-03-24T02:15:07.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 pool 2026-03-24T02:15:07.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2026-03-24T02:15:07.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent/ns1 pool 2026-03-24T02:15:07.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -z ']' 2026-03-24T02:15:07.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -z ']' 2026-03-24T02:15:07.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ peer_add cluster2 mirror cluster1 2026-03-24T02:15:07.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:15:07.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:15:07.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:15:07.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:15:07.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local client_cluster=cluster1 2026-03-24T02:15:07.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:15:07.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_cluster=cluster1 2026-03-24T02:15:07.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local uuid_var_name 2026-03-24T02:15:07.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:15:07.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local error_code 2026-03-24T02:15:07.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local peer_uuid 2026-03-24T02:15:07.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 16 32 2026-03-24T02:15:07.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:15:07.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror cluster1 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ peer_uuid=a03b4380-eaec-434a-87e1-101dd17505a9 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=0 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 -eq 17 ']' 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 0 -eq 0 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ peer_add cluster2 mirror_parent cluster1 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror_parent 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local client_cluster=cluster1 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_cluster=cluster1 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local uuid_var_name 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local error_code 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local peer_uuid 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 16 32 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:15:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent cluster1 2026-03-24T02:15:07.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ peer_uuid=9412cb78-0b46-475c-a2f9-74bd4bc8fcce 2026-03-24T02:15:07.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=0 2026-03-24T02:15:07.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:15:07.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 -eq 17 ']' 2026-03-24T02:15:07.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 0 -eq 0 2026-03-24T02:15:07.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:15:07.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:15:07.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:15:07.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ start_mirrors cluster1 2026-03-24T02:15:07.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:15:07.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:15:07.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ start_mirror cluster1:0 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ start_mirror cluster1:1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ start_mirror cluster1:2 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 1 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ start_mirror cluster1:3 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:15:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 1 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ start_mirrors cluster2 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ start_mirror cluster2:0 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2:0 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster2:0 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster2 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster2:0 ']' 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster2 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster2 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 1 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ start_mirror cluster2:1 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2:1 2026-03-24T02:15:07.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster2:1 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster2 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster2:1 ']' 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster2 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster2 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 1 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ start_mirror cluster2:2 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2:2 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster2:2 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster2 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster2:2 ']' 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster2 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster2 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 1 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ start_mirror cluster2:3 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2:3 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster2:3 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster2 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster2:3 ']' 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster2 2026-03-24T02:15:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster2 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 1 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ testlog 'TEST: add image and test replay after client crashes' 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ tee -a /tmp/tmp.l3GEA596QR/rbd-mirror.test.log 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ date '+%F %T' 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo 2026-03-24 02:15:07 TEST: add image and test replay after client crashes 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:2026-03-24 02:15:07 TEST: add image and test replay after client crashes 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=test 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror test journal 512M 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror test 512M 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 512M ']' 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=512M 2026-03-24T02:15:07.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:15:07.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/test 2026-03-24T02:15:07.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:15:07.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:15:07.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:15:07.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:15:07.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:15:07.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-24T02:15:07.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:15:07.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:15:07.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:15:07.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-24T02:15:07.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:15:07.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:15:07.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:15:07.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:15:07.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:15:07.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:15:07.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:15:08.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-24T02:15:08.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:15:08.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:15:08.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:15:08.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:15:08.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:15:08.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:15:08.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-24T02:15:08.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:15:08.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:15:08.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:08.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:15:08.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:15:08.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:15:08.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:15:08.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:15:08.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:15:08.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:15:08.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:08.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-24T02:15:08.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:15:08.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:08.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:08.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:15:08.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:15:08.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:08.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:08.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:08.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:15:08.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:15:08.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:08.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:08.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:15:08.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:15:08.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:15:08.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:15:08.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:08.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:08.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:15:08.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:15:08.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:08.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:08.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:08.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:15:08.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:15:08.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:08.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:08.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:15:08.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:15:08.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:15:08.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:08.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:08.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:08.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-24T02:15:08.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:08.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:08.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:08.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:08.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-24T02:15:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:15:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:15:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:15:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:15:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:15:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:15:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:15:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:15:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:15:08.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:08.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:08.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:15:08.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:15:08.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:08.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:08.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:08.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:15:08.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:15:08.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:08.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:08.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:15:08.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:15:08.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:15:08.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:15:08.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:15:08.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:15:08.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-24T02:15:08.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:08.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:08.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:08.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:08.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-03-24T02:15:08.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:15:08.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:08.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:08.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:15:08.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:15:08.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:08.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:08.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:08.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:15:08.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:15:08.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:08.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:08.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:15:08.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:15:08.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:15:08.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:15:08.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:08.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:08.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:15:08.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:15:08.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:08.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:08.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:08.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:15:08.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:15:08.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:08.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:08.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:15:08.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:15:08.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:15:08.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:15:08.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:15:08.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:15:08.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-03-24T02:15:08.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:08.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:08.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-03-24T02:15:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:15:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:15:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:15:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:15:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:15:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:15:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:15:08.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:15:08.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:15:08.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:08.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:08.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:15:08.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:15:08.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:08.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:08.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:08.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:15:08.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:15:08.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:08.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:08.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:15:08.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:15:08.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:15:08.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:15:08.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:15:08.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:15:08.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-03-24T02:15:08.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:08.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:08.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:08.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:15:08.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 1 2026-03-24T02:15:09.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:15:09.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:15:09.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:09.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-24T02:15:09.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:15:09.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:09.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:09.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:15:09.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:15:09.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:09.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:09.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:09.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:15:09.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:15:09.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:09.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:09.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:15:09.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:15:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:15:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:15:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:15:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:15:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:15:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:15:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:15:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:15:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:15:09.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:09.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:09.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:09.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-24T02:15:09.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:09.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:09.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:09.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:09.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-24T02:15:09.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:15:09.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:09.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:09.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:15:09.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:09.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:09.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:15:09.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:15:09.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:15:09.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:15:09.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:15:09.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:15:09.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-24T02:15:09.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:09.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:09.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:09.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:09.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-03-24T02:15:09.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:15:09.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:09.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:09.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:15:09.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:15:09.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:09.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:09.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:09.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:15:09.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:15:09.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:15:09.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:15:09.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:15:09.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:15:09.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-03-24T02:15:09.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:09.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:09.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:09.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:09.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-03-24T02:15:09.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:15:09.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:09.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:09.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:15:09.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:15:09.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:09.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:09.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:09.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:15:09.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:15:09.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:15:09.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:15:09.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:15:09.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:15:09.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-03-24T02:15:09.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:09.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:09.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:09.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:15:09.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 2 2026-03-24T02:15:11.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:15:11.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:15:11.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-24T02:15:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:15:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:15:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:15:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:15:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:15:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:15:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:15:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:15:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:15:11.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:11.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:11.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:15:11.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:15:11.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:11.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:11.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:11.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:15:11.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:15:11.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:11.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:11.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:15:11.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:15:11.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:15:11.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:11.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:11.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:11.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-24T02:15:11.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:11.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:11.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:11.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:11.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-24T02:15:11.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:15:11.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:11.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:11.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:15:11.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:11.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:11.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:15:11.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:15:11.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:15:11.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:15:11.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:15:11.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:15:11.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-24T02:15:12.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:12.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:12.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:12.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:12.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-03-24T02:15:12.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:15:12.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:12.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:12.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:15:12.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:15:12.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:12.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:12.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:12.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:15:12.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:15:12.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:12.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:12.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:15:12.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:15:12.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:15:12.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:15:12.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:12.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:12.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:15:12.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:15:12.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:12.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:12.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:12.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:15:12.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:15:12.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:12.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:12.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:15:12.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:15:12.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:15:12.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:15:12.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:15:12.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:15:12.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-03-24T02:15:12.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:12.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:12.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:15:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:15:12.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:12.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:12.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:12.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:15:12.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:15:12.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:12.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:12.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:15:12.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:15:12.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:15:12.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:15:12.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:15:12.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:15:12.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-03-24T02:15:12.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:12.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:12.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:12.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:15:12.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 4 2026-03-24T02:15:16.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:15:16.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:15:16.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:16.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:16.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:15:16.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:15:16.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:16.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:16.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:16.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:15:16.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:15:16.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:16.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:16.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:15:16.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:15:16.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:15:16.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:16.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:16.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:16.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-24T02:15:16.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:16.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:16.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:16.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:16.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-24T02:15:16.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:15:16.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:16.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:16.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:15:16.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:15:16.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:16.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:16.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:16.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:15:16.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:15:16.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:16.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:16.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:15:16.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:15:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:15:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:15:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:15:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:15:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:15:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:15:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:15:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:15:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:15:16.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:15:16.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:15:16.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:15:16.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-24T02:15:16.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:16.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:16.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:15:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:15:16.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:16.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:16.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:15:16.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:15:16.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:16.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:16.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:16.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:15:16.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:15:16.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:16.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:16.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:15:16.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:15:16.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:15:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:15:16.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:15:16.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:15:16.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-03-24T02:15:16.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:16.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:16.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-03-24T02:15:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:15:16.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:15:16.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:15:16.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:15:16.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:15:16.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:15:16.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-03-24T02:15:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:16.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:15:16.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 8 2026-03-24T02:15:18.183 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-03-24T02:15:18.183 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-03-24T02:15:18.183 INFO:tasks.rbd_mirror_thrash:waiting for 2.9597821035640925 secs before reviving daemons 2026-03-24T02:15:18.184 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:15:18.177+0000 7fed97e2d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 26417) UID: 1000 2026-03-24T02:15:21.144 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-03-24T02:15:21.144 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:15:21.144 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-03-24T02:15:21.144 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-03-24T02:15:21.144 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-03-24T02:15:21.145 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-03-24T02:15:21.145 INFO:tasks.rbd_mirror_thrash:waiting for 74.01027002582539 secs before thrashing 2026-03-24T02:15:24.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:15:24.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:15:24.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:15:24.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:15:24.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:24.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:24.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:24.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-24T02:15:24.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:24.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:24.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:24.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:24.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-24T02:15:24.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:15:24.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:24.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:24.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:15:24.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:15:24.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:24.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:24.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:24.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:15:24.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:15:24.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:24.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:24.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:15:24.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:15:24.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:15:24.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:15:24.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:24.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:24.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:15:24.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:15:24.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:24.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:24.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:24.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:15:24.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:15:24.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:24.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:24.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:15:24.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:15:24.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:15:24.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:15:24.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:15:24.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:15:24.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-24T02:15:24.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:24.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:24.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:15:24.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:15:24.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:15:24.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:15:24.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:15:24.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:15:24.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-03-24T02:15:24.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:24.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:24.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:24.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:24.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-03-24T02:15:24.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:15:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:15:24.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:15:24.731 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:15:24.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:15:24.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:15:24.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-03-24T02:15:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-24T02:15:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:15:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:15:24.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:15:24.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 8 2026-03-24T02:15:31.584 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-03-24T02:15:31.584 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-03-24T02:15:31.584 INFO:tasks.rbd_mirror_thrash:waiting for 3.588120948358905 secs before reviving daemons 2026-03-24T02:15:31.585 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:15:31.585+0000 7fc284775640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 26425) UID: 1000 2026-03-24T02:15:32.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:15:32.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:15:32.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:15:32.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-24T02:15:32.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:15:32.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:15:32.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:15:32.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:15:32.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:15:32.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:15:32.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:15:32.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:15:32.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:15:32.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:15:32.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:15:32.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:32.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:15:32.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:15:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:15:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:15:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:15:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:15:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:15:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:15:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:15:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:15:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:15:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:15:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:15:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:15:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:15:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:15:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:15:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:15:32.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:32.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:32.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:15:32.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-24T02:15:32.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:15:32.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:15:32.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:15:32.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:15:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:15:32.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:15:32.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:15:32.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:15:32.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ clean_snap_name= 2026-03-24T02:15:32.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 1 10 2026-03-24T02:15:32.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:15:32.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror test 2026-03-24T02:15:32.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:15:32.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:15:32.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:15:32.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:15:32.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=6 2026-03-24T02:15:32.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:15:32.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-03-24T02:15:33.224 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:15:33.217+0000 7fe8af94d980 -1 rbd::mirror::Mirror: 0x55ea250a7200 update_pool_replayers: restarting failed pool replayer for uuid: dddca223-77d9-43ad-a869-0470d458d5b2 cluster: cluster2 client: client.mirror 2026-03-24T02:15:33.234 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:15:33.229+0000 7f414e356980 -1 rbd::mirror::Mirror: 0x55764b86d200 update_pool_replayers: restarting failed pool replayer for uuid: dddca223-77d9-43ad-a869-0470d458d5b2 cluster: cluster2 client: client.mirror 2026-03-24T02:15:33.410 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:15:33.405+0000 7f504c974980 -1 rbd::mirror::Mirror: 0x55ee0a22d200 update_pool_replayers: restarting failed pool replayer for uuid: dddca223-77d9-43ad-a869-0470d458d5b2 cluster: cluster2 client: client.mirror 2026-03-24T02:15:33.775 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:15:33.769+0000 7f414e356980 -1 rbd::mirror::Mirror: 0x55764b86d200 update_pool_replayers: restarting failed pool replayer for uuid: b711d228-8fb4-4ed9-91b8-89c31ba9f34d cluster: cluster2 client: client.mirror 2026-03-24T02:15:33.776 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:15:33.769+0000 7fe8af94d980 -1 rbd::mirror::Mirror: 0x55ea250a7200 update_pool_replayers: restarting failed pool replayer for uuid: b711d228-8fb4-4ed9-91b8-89c31ba9f34d cluster: cluster2 client: client.mirror 2026-03-24T02:15:33.824 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:15:33.821+0000 7f504c974980 -1 rbd::mirror::Mirror: 0x55ee0a22d200 update_pool_replayers: restarting failed pool replayer for uuid: b711d228-8fb4-4ed9-91b8-89c31ba9f34d cluster: cluster2 client: client.mirror 2026-03-24T02:15:34.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 20755456 bytes 2026-03-24T02:15:35.173 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-03-24T02:15:35.173 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:15:35.173 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-03-24T02:15:35.174 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-03-24T02:15:35.174 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-03-24T02:15:35.175 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-03-24T02:15:35.175 INFO:tasks.rbd_mirror_thrash:waiting for 118.9217706567989 secs before thrashing 2026-03-24T02:15:35.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 43224064 bytes 2026-03-24T02:15:35.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 66637824 bytes 2026-03-24T02:15:36.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:400 IOs, 67269632 bytes 2026-03-24T02:15:38.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:500 IOs, 105622528 bytes 2026-03-24T02:15:38.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:15:38.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:15:38.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:15:38.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:15:38.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-24T02:15:38.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:15:38.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:15:38.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:15:38.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:15:38.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:15:38.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:15:38.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:15:38.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:15:39.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-24T02:15:39.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:15:39.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:15:39.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:15:39.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:15:39.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:15:39.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:15:39.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:15:39.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:15:39.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-24T02:15:39.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='test: 2026-03-24T02:15:39.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: global_id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:15:39.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+replaying 2026-03-24T02:15:39.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-03-24T02:15:39.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: service: mirror.0 on vm07 2026-03-24T02:15:39.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:15:25 2026-03-24T02:15:39.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: peer_sites: 2026-03-24T02:15:39.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: name: cluster2 2026-03-24T02:15:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+stopped 2026-03-24T02:15:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: local image is primary 2026-03-24T02:15:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:15:36' 2026-03-24T02:15:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ state: .*up+replaying' 2026-03-24T02:15:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: state: up+replaying 2026-03-24T02:15:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ description: .*' 2026-03-24T02:15:39.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-03-24T02:15:39.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:15:39.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:15:39.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^up+' 2026-03-24T02:15:39.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:15:39.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep 'service: *mirror..* on ' 2026-03-24T02:15:39.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: service: mirror.0 on vm07 2026-03-24T02:15:39.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update 2026-03-24T02:15:39.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-24T02:15:39.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:15:25' 2026-03-24T02:15:39.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-24 02:15:25' 2026-03-24T02:15:39.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:15:39.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:15:39.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:15:39.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:15:39.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'prev_last_update=2026-03-24 02:15:25' 2026-03-24T02:15:39.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:15:39.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:15:39.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-24T02:15:39.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='OKOKOK142234307mirror.1vm0720.2.0-712-g70f8415bfalseOK42254303mirror.2vm0720.2.0-712-g70f8415bfalseOK42264305mirror.3vm0720.2.0-712-g70f8415bfalseOK42944297mirror.0vm0720.2.0-712-g70f8415btrueOKtest2d4a20c9-f6e9-4ce8-9d64-84febc265629up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42944297mirror.0vm072026-03-24 02:15:25cluster26c644ee3-7db1-4d97-9283-48b1f8a82096up+stoppedlocal image is primary2026-03-24 02:15:36' 2026-03-24T02:15:39.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update state 2026-03-24T02:15:39.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-24T02:15:39.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:15:25' 2026-03-24T02:15:39.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-24T02:15:39.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=up+replaying 2026-03-24T02:15:39.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:15:39.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep up+replaying 2026-03-24T02:15:40.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:15:40.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:15:40.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:15:40.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap1 2026-03-24T02:15:40.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap1 2026-03-24T02:15:40.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:15:40.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:15:40.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:15:40.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap1 2026-03-24T02:15:40.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:16:09.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-24T02:16:09.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:09.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:16:09.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:09.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-24T02:16:09.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:09.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:16:09.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:09.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:16:09.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:16:09.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:16:09.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:16:10.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-24T02:16:10.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:10.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:16:10.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:10.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:16:10.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:16:10.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:16:10.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-24T02:16:10.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:16:10.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:16:10.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:16:10.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:16:10.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:16:10.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:16:10.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:16:10.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:16:10.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:16:10.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:16:10.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:16:10.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-24T02:16:10.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:16:10.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:16:10.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:16:10.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:16:10.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:16:10.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:16:10.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:16:10.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:16:10.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:16:10.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:16:10.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:16:10.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:16:10.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:10.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:10.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:10.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-24T02:16:10.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:16:10.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:16:10.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:16:10.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:16:10.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:16:10.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:16:10.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:16:10.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:16:10.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:10.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:16:10.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:10.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:16:10.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:16:10.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:16:10.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:16:10.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:16:10.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:16:10.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:10.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:16:10.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:16:10.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:16:10.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:16:10.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:16:10.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:16:10.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:16:10.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:16:10.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:16:10.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:16:10.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:16:10.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:16:10.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:16:10.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:16:10.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:16:10.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:16:10.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:16:10.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:16:10.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:16:10.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:16:10.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:16:10.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:16:10.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:16:10.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:16:10.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:16:10.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:16:10.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:16:10.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:16:10.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:16:10.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:16:10.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:16:10.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:16:10.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:16:10.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:16:10.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:16:10.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:16:10.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:16:10.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:16:10.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:10.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:10.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:10.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:16:10.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:16:10.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:16:10.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:10.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:10.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:10.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:16:10.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:10.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:10.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:10.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:16:10.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:16:10.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:16:10.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:16:10.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:10.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:16:10.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:10.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:16:10.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:16:10.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:16:10.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=2, entry_tid=6858], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:10.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=6, tag_tid=2, entry_tid=6858], [object_number=5, tag_tid=2, entry_tid=6857], [object_number=4, tag_tid=2, entry_tid=6856], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:10.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:10.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:16:10.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:16:10.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:10.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:10.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:10.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:16:10.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:10.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:10.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:10.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:16:10.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:16:10.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:16:10.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:16:10.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:10.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:16:10.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:10.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:16:10.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:16:10.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:16:10.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=2, entry_tid=6858], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:10.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=6, tag_tid=2, entry_tid=6858], [object_number=5, tag_tid=2, entry_tid=6857], [object_number=4, tag_tid=2, entry_tid=6856], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:10.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:10.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=6, tag_tid=2, entry_tid=6858' 2026-03-24T02:16:10.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=6, tag_tid=2, entry_tid=6858' 2026-03-24T02:16:10.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=6, tag_tid=2, entry_tid=6858' '!=' '' 2026-03-24T02:16:10.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:16:10.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=6, tag_tid=2, entry_tid=6858' = '' 2026-03-24T02:16:10.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=6, tag_tid=2, entry_tid=6858' 2026-03-24T02:16:10.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:16:10.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:16:10.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:16:10.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:16:10.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=6, tag_tid=2, entry_tid=6858' 2026-03-24T02:16:10.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:16:10.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:16:10.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:16:10.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:16:10.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:16:10.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:16:10.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:16:10.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=6, tag_tid=2, entry_tid=6858' 2026-03-24T02:16:10.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:16:10.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:16:10.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=6858 2026-03-24T02:16:10.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 6858 2026-03-24T02:16:10.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:16:10.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:16:10.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:16:11.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:16:11.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:11.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:16:11.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:11.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:16:11.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:16:11.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:16:11.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:16:11.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:16:11.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:16:11.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:11.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:16:11.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:16:11.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:16:11.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:16:11.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:16:11.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:16:11.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:16:11.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:16:11.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:16:11.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:16:11.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:16:11.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:16:11.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:16:11.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:16:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:16:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:16:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:16:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:16:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:16:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:16:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:16:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:16:11.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:11.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:16:11.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:11.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:16:11.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:16:11.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:16:11.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=2, entry_tid=6858], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:11.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=6, tag_tid=2, entry_tid=6858], [object_number=5, tag_tid=2, entry_tid=6857], [object_number=4, tag_tid=2, entry_tid=6856], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:11.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:16:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:16:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:16:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:11.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:11.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:11.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:16:11.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:16:11.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:16:11.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:16:11.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:11.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:16:11.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:11.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:16:11.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:16:11.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:16:11.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=2, entry_tid=6858], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:11.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=6, tag_tid=2, entry_tid=6858], [object_number=5, tag_tid=2, entry_tid=6857], [object_number=4, tag_tid=2, entry_tid=6856], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:11.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:11.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=6, tag_tid=2, entry_tid=6858' 2026-03-24T02:16:11.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=6, tag_tid=2, entry_tid=6858' 2026-03-24T02:16:11.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=6, tag_tid=2, entry_tid=6858' '!=' 'object_number=6, tag_tid=2, entry_tid=6858' 2026-03-24T02:16:11.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:16:11.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:16:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:16:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:16:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:16:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:16:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:16:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:16:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:16:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:16:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:16:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:16:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:16:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:16:11.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:16:11.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:16:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:16:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:16:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:16:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:16:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:16:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:16:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:16:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:16:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:16:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:16:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:16:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:16:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:16:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:16:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:16:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:16:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:16:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:16:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:16:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:16:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:16:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:16:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:16:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:16:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:16:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:16:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:16:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:16:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:16:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:16:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:16:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:16:11.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:11.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:11.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:11.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:16:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:16:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:16:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:16:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:16:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:16:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:16:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:16:11.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:11.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:16:11.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:11.832 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:16:11.833 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:16:11.833 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:16:11.833 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=2, entry_tid=6858], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:11.833 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=6, tag_tid=2, entry_tid=6858], [object_number=5, tag_tid=2, entry_tid=6857], [object_number=4, tag_tid=2, entry_tid=6856], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:11.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:11.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:16:11.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:16:11.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:11.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:11.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:11.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:16:11.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:11.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:11.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:11.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:16:11.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:16:11.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:16:11.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:16:11.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:11.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:16:11.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:11.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:16:11.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:16:11.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:16:11.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=2, entry_tid=6858], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:11.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=6, tag_tid=2, entry_tid=6858], [object_number=5, tag_tid=2, entry_tid=6857], [object_number=4, tag_tid=2, entry_tid=6856], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:11.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:11.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=6, tag_tid=2, entry_tid=6858' 2026-03-24T02:16:11.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=6, tag_tid=2, entry_tid=6858' 2026-03-24T02:16:11.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=6, tag_tid=2, entry_tid=6858' '!=' 'object_number=6, tag_tid=2, entry_tid=6858' 2026-03-24T02:16:11.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:16:11.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:16:12.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:16:12.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:12.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:16:12.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:12.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:16:12.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:16:12.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:16:12.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:16:12.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:16:12.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:16:12.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:12.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:16:12.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:16:12.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:16:12.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:16:12.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:16:12.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:16:12.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:16:12.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:16:12.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:16:12.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:16:12.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:16:12.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:12.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:12.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:12.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:16:12.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:16:12.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:16:12.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:12.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:12.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:12.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:16:12.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:12.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:12.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:12.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:16:12.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:16:12.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:16:12.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:16:12.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:12.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:16:12.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:12.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:16:12.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:16:12.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:16:12.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=2, entry_tid=6858], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:12.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=6, tag_tid=2, entry_tid=6858], [object_number=5, tag_tid=2, entry_tid=6857], [object_number=4, tag_tid=2, entry_tid=6856], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:12.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:12.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:16:12.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:16:12.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:12.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:12.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:12.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:16:12.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:12.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:12.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:12.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:16:12.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:16:12.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:16:12.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:16:12.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:12.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:16:12.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:12.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:16:12.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:16:12.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:16:12.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=2, entry_tid=6858], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:12.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=6, tag_tid=2, entry_tid=6858], [object_number=5, tag_tid=2, entry_tid=6857], [object_number=4, tag_tid=2, entry_tid=6856], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:12.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:12.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=6, tag_tid=2, entry_tid=6858' 2026-03-24T02:16:12.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=6, tag_tid=2, entry_tid=6858' 2026-03-24T02:16:12.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=6, tag_tid=2, entry_tid=6858' '!=' 'object_number=6, tag_tid=2, entry_tid=6858' 2026-03-24T02:16:12.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:16:12.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:16:14.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:16:14.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:14.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:16:14.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:14.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:16:14.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:16:14.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:16:14.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:16:14.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:16:14.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:16:14.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:14.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:16:14.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:16:14.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:16:14.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:16:14.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:16:14.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:16:14.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:16:14.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:16:14.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:16:14.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:16:14.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:16:14.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:16:14.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:16:14.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:16:14.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:16:14.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:16:14.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:16:14.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:16:14.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:14.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:14.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:14.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:16:14.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:16:14.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:16:14.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:14.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:14.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:14.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:16:14.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:14.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:14.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:14.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:16:14.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:16:14.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:16:14.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:16:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:16:14.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:16:14.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:16:14.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:16:14.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=2, entry_tid=6858], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:14.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=6, tag_tid=2, entry_tid=6858], [object_number=5, tag_tid=2, entry_tid=6857], [object_number=4, tag_tid=2, entry_tid=6856], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:14.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:14.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:16:14.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:16:14.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:14.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:14.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:14.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:16:14.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:14.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:14.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:14.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:16:14.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:16:14.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:16:14.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:16:14.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:14.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:16:14.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:14.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:16:14.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:16:14.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:16:14.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=2, entry_tid=6858], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:14.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=6, tag_tid=2, entry_tid=6858], [object_number=5, tag_tid=2, entry_tid=6857], [object_number=4, tag_tid=2, entry_tid=6856], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:14.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:14.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=6, tag_tid=2, entry_tid=6858' 2026-03-24T02:16:14.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=6, tag_tid=2, entry_tid=6858' 2026-03-24T02:16:14.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=6, tag_tid=2, entry_tid=6858' '!=' 'object_number=6, tag_tid=2, entry_tid=6858' 2026-03-24T02:16:14.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:16:14.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:16:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:16:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:16:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:16:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:16:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:16:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:16:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:16:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:16:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:16:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:16:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:16:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:16:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:16:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:16:16.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:16:16.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:16:16.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:16:16.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:16:16.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:16:16.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:16:16.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:16:16.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:16:16.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:16:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:16:16.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:16.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:16.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:16:16.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:16:16.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:16:16.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:16:16.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:16.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:16.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:16.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:16:16.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:16.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:16.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:16.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:16:16.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:16:16.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:16:16.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:16:16.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:16.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:16:16.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:16.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:16:16.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:16:16.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:16:16.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=2, entry_tid=6858], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:16.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=2, entry_tid=6858], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:16.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:16.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:16:16.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:16:16.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:16.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:16.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:16.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:16:16.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:16:16.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:16:16.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:16:16.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:16:16.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:16:16.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:16:16.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:16:16.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:16.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:16:16.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:16.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:16:16.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:16:16.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:16:16.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=2, entry_tid=6858], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:16.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=2, entry_tid=6858], [object_number=7, tag_tid=2, entry_tid=6855]]], state=connected] 2026-03-24T02:16:16.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:16:16.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:16:16.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:16:16.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:16:16.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror test snap1 2026-03-24T02:16:16.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:16.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:16:16.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:16.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap1 2026-03-24T02:16:16.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:16:16.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:16:16.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:16:17.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/test@snap1 2026-03-24T02:16:17.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'test': 2026-03-24T02:16:17.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:16:17.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:16:17.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:16:17.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c9689d5e36 2026-03-24T02:16:17.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c9689d5e36 2026-03-24T02:16:17.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:16:17.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:16:17.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:16:17.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:16:17.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:15:23 2026 2026-03-24T02:16:17.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:15:23 2026 2026-03-24T02:16:17.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:15:23 2026 2026-03-24T02:16:17.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:16:17.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c9689d5e36 2026-03-24T02:16:17.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:16:17.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:16:17.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:16:17.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:16:17.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:16:17.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:16:17.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap1 2026-03-24T02:16:17.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:16:17.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:17.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap1 2026-03-24T02:16:17.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:16:17.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:16:17.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:16:17.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:16:17.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:16:19.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:16:19.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:16:20.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:16:20.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:16:21.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:16:21.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:16:21.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ clean_snap_name=snap1-clean 2026-03-24T02:16:21.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap1-clean 2026-03-24T02:16:21.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:16:21.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:16:21.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:21.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap1-clean 2026-03-24T02:16:21.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1-clean --debug-rbd=20 --debug-journaler=20 2026-03-24T02:16:21.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:16:21.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror test 2026-03-24T02:16:21.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:16:21.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:16:21.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:21.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:16:21.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=7 2026-03-24T02:16:21.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:16:21.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-03-24T02:16:24.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 56151040 bytes 2026-03-24T02:16:25.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 93501440 bytes 2026-03-24T02:16:28.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 184293376 bytes 2026-03-24T02:16:28.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:16:28.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:16:28.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:16:28.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:16:28.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-24T02:16:28.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:28.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:16:28.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:28.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:16:28.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:16:28.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:16:28.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:16:28.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:16:29.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-24T02:16:29.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:29.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:16:29.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:29.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:16:29.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:16:29.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:16:29.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:16:29.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:16:30.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-24T02:16:30.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='test: 2026-03-24T02:16:30.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: global_id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:16:30.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+replaying 2026-03-24T02:16:30.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: replaying, {"bytes_per_second":1719212.23,"entries_behind_primary":0,"entries_per_second":109.27,"non_primary_position":{"entry_tid":1,"object_number":5,"tag_tid":3},"primary_position":{"entry_tid":1,"object_number":5,"tag_tid":3}} 2026-03-24T02:16:30.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: service: mirror.0 on vm07 2026-03-24T02:16:30.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:16:21 2026-03-24T02:16:30.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: peer_sites: 2026-03-24T02:16:30.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: name: cluster2 2026-03-24T02:16:30.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+stopped 2026-03-24T02:16:30.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: local image is primary 2026-03-24T02:16:30.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:16:04' 2026-03-24T02:16:30.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ state: .*up+replaying' 2026-03-24T02:16:30.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: state: up+replaying 2026-03-24T02:16:30.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ description: .*' 2026-03-24T02:16:30.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: description: replaying, {"bytes_per_second":1719212.23,"entries_behind_primary":0,"entries_per_second":109.27,"non_primary_position":{"entry_tid":1,"object_number":5,"tag_tid":3},"primary_position":{"entry_tid":1,"object_number":5,"tag_tid":3}} 2026-03-24T02:16:30.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:16:30.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:16:30.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^up+' 2026-03-24T02:16:30.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:16:30.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep 'service: *mirror..* on ' 2026-03-24T02:16:30.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: service: mirror.0 on vm07 2026-03-24T02:16:30.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update 2026-03-24T02:16:30.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-24T02:16:30.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:16:21' 2026-03-24T02:16:30.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-24 02:16:21' 2026-03-24T02:16:30.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:16:30.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:16:30.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:30.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:16:30.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'prev_last_update=2026-03-24 02:16:21' 2026-03-24T02:16:30.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:16:30.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:16:30.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-24T02:16:30.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='OKOKOK142234307mirror.1vm0720.2.0-712-g70f8415bfalseOK42254303mirror.2vm0720.2.0-712-g70f8415bfalseOK42264305mirror.3vm0720.2.0-712-g70f8415bfalseOK42944297mirror.0vm0720.2.0-712-g70f8415btrueOKtest2d4a20c9-f6e9-4ce8-9d64-84febc265629up+replayingreplaying, {"bytes_per_second":1719212.23,"entries_behind_primary":0,"entries_per_second":109.27,"non_primary_position":{"entry_tid":1,"object_number":5,"tag_tid":3},"primary_position":{"entry_tid":1,"object_number":5,"tag_tid":3}}42944297mirror.0vm072026-03-24 02:16:21cluster26c644ee3-7db1-4d97-9283-48b1f8a82096up+stoppedlocal image is primary2026-03-24 02:16:04' 2026-03-24T02:16:30.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update state 2026-03-24T02:16:30.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-24T02:16:30.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:16:21' 2026-03-24T02:16:30.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-24T02:16:30.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=up+replaying 2026-03-24T02:16:30.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:16:30.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep up+replaying 2026-03-24T02:16:30.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:16:30.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:16:30.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:16:30.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap2 2026-03-24T02:16:30.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap2 2026-03-24T02:16:30.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:16:30.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:16:30.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:16:30.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap2 2026-03-24T02:16:30.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:16:35.156 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.280109005413155) > weight (0.25) 2026-03-24T02:16:35.156 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-03-24T02:16:35.156 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-03-24T02:16:35.156 INFO:tasks.rbd_mirror_thrash:waiting for 2.1887195543045634 secs before reviving daemons 2026-03-24T02:16:35.157 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:16:35.153+0000 7f504992f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 26419) UID: 1000 2026-03-24T02:16:37.346 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-03-24T02:16:37.346 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:16:37.346 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-03-24T02:16:37.347 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-03-24T02:16:37.347 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-03-24T02:16:37.348 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-03-24T02:16:37.348 INFO:tasks.rbd_mirror_thrash:waiting for 107.57910730052204 secs before thrashing 2026-03-24T02:17:00.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-24T02:17:00.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:00.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:00.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:00.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-24T02:17:00.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:00.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:00.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:00.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:17:00.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:17:00.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:00.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:17:01.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-24T02:17:01.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:01.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:01.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:01.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:17:01.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:17:01.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:17:01.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-24T02:17:01.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:17:01.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:17:01.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:17:01.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:01.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:17:01.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:17:01.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:01.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:17:01.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:01.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:17:01.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:01.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:17:01.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:01.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:01.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:01.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:01.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-24T02:17:01.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:17:01.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:17:01.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:17:01.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:17:01.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:17:01.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:17:01.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:01.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:01.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:01.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:01.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:01.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:01.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:01.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:01.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:01.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:01.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:01.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:01.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:01.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:01.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:01.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:01.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:01.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:01.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:01.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:01.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:01.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:01.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:01.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:01.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:01.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:01.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:01.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:01.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:01.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:01.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:01.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:01.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:01.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:01.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:01.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:01.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:01.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:01.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:01.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:01.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 4 2026-03-24T02:17:01.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 4 2026-03-24T02:17:01.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:01.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=6, entry_tid=1], [object_number=16, tag_tid=6, entry_tid=0], [object_number=19, tag_tid=5, entry_tid=12615], [object_number=18, tag_tid=5, entry_tid=12614]]], state=connected] 2026-03-24T02:17:01.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=17, tag_tid=6, entry_tid=1], [object_number=16, tag_tid=6, entry_tid=0], [object_number=19, tag_tid=5, entry_tid=12615], [object_number=18, tag_tid=5, entry_tid=12614]]], state=connected] 2026-03-24T02:17:01.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:01.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=17, tag_tid=6, entry_tid=1' 2026-03-24T02:17:01.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:01.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:01.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:01.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:01.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:01.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:01.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:01.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:01.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:01.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:01.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:01.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:01.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:01.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:01.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:01.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 4 2026-03-24T02:17:01.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 4 2026-03-24T02:17:01.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:01.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=6, entry_tid=1], [object_number=16, tag_tid=6, entry_tid=0], [object_number=19, tag_tid=5, entry_tid=12615], [object_number=18, tag_tid=5, entry_tid=12614]]], state=connected] 2026-03-24T02:17:01.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=17, tag_tid=6, entry_tid=1], [object_number=16, tag_tid=6, entry_tid=0], [object_number=19, tag_tid=5, entry_tid=12615], [object_number=18, tag_tid=5, entry_tid=12614]]], state=connected] 2026-03-24T02:17:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:01.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=17, tag_tid=6, entry_tid=1' 2026-03-24T02:17:01.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=17, tag_tid=6, entry_tid=1' -a 'object_number=17, tag_tid=6, entry_tid=1' = 'object_number=17, tag_tid=6, entry_tid=1' 2026-03-24T02:17:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror test snap2 2026-03-24T02:17:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap2 2026-03-24T02:17:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:17:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:17:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:17:02.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/test@snap2 2026-03-24T02:17:02.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'test': 2026-03-24T02:17:02.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:17:02.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:17:02.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 3 2026-03-24T02:17:02.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c9689d5e36 2026-03-24T02:17:02.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c9689d5e36 2026-03-24T02:17:02.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:17:02.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:17:02.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:17:02.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:17:02.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:15:23 2026 2026-03-24T02:17:02.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:15:23 2026 2026-03-24T02:17:02.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:16:26 2026 2026-03-24T02:17:02.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:17:02.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c9689d5e36 2026-03-24T02:17:02.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:17:02.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:17:02.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:17:02.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:17:02.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:02.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n snap1-clean ']' 2026-03-24T02:17:02.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap1-clean 2026-03-24T02:17:02.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:02.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:02.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap1-clean 2026-03-24T02:17:02.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:17:02.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:17:02.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:17:02.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:17:02.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:17:03.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:17:03.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:17:03.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:17:04.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:17:04.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:17:04.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:04.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap2 2026-03-24T02:17:04.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:04.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:04.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap2 2026-03-24T02:17:04.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:17:04.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:17:04.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:17:04.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:17:04.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:17:04.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:17:04.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:17:05.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:17:05.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:17:05.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:17:05.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:05.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ clean_snap_name=snap2-clean 2026-03-24T02:17:05.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap2-clean 2026-03-24T02:17:05.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:17:05.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:05.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:05.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap2-clean 2026-03-24T02:17:05.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2-clean --debug-rbd=20 --debug-journaler=20 2026-03-24T02:17:05.789 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:17:05.785+0000 7f08511d5640 -1 bdev(0x5587822c5400 /var/lib/ceph/osd/cluster2-2/block) aio_submit retries 2 2026-03-24T02:17:06.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:17:06.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror test 2026-03-24T02:17:06.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:17:06.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:06.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:06.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:17:06.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=9 2026-03-24T02:17:06.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:17:06.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-03-24T02:17:08.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 47270912 bytes 2026-03-24T02:17:09.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 81388544 bytes 2026-03-24T02:17:10.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 98106368 bytes 2026-03-24T02:17:10.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:400 IOs, 98710528 bytes 2026-03-24T02:17:12.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:500 IOs, 152520704 bytes 2026-03-24T02:17:14.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:600 IOs, 216232960 bytes 2026-03-24T02:17:15.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:17:15.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:17:15.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:17:15.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:15.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-24T02:17:15.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:15.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:15.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:15.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:17:15.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:17:15.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:17:15.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:15.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:17:16.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-24T02:17:16.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:16.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:16.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:16.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:17:16.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:17:16.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:17:16.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:17:16.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:17:16.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-24T02:17:16.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='test: 2026-03-24T02:17:16.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: global_id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:17:16.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+replaying 2026-03-24T02:17:16.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: replaying, {"bytes_per_second":3850258.83,"entries_behind_primary":2380,"entries_per_second":239.3,"non_primary_position":{"entry_tid":5267,"object_number":11,"tag_tid":5},"primary_position":{"entry_tid":7647,"object_number":15,"tag_tid":5},"seconds_until_synced":9} 2026-03-24T02:17:16.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: service: mirror.0 on vm07 2026-03-24T02:17:16.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:16:51 2026-03-24T02:17:16.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: peer_sites: 2026-03-24T02:17:16.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: name: cluster2 2026-03-24T02:17:16.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+stopped 2026-03-24T02:17:16.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: local image is primary 2026-03-24T02:17:16.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:17:04' 2026-03-24T02:17:16.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ state: .*up+replaying' 2026-03-24T02:17:16.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: state: up+replaying 2026-03-24T02:17:16.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ description: .*' 2026-03-24T02:17:16.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: description: replaying, {"bytes_per_second":3850258.83,"entries_behind_primary":2380,"entries_per_second":239.3,"non_primary_position":{"entry_tid":5267,"object_number":11,"tag_tid":5},"primary_position":{"entry_tid":7647,"object_number":15,"tag_tid":5},"seconds_until_synced":9} 2026-03-24T02:17:16.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:17:16.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:17:16.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^up+' 2026-03-24T02:17:16.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:17:16.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep 'service: *mirror..* on ' 2026-03-24T02:17:16.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: service: mirror.0 on vm07 2026-03-24T02:17:16.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update 2026-03-24T02:17:16.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-24T02:17:16.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:16:51' 2026-03-24T02:17:16.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-24 02:16:51' 2026-03-24T02:17:16.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:16.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:16.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:16.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:17:16.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'prev_last_update=2026-03-24 02:16:51' 2026-03-24T02:17:16.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:17:16.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:17:16.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-24T02:17:16.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='OKOKOK142254303mirror.2vm0720.2.0-712-g70f8415bfalseOK42264305mirror.3vm0720.2.0-712-g70f8415bfalseOK42944297mirror.0vm0720.2.0-712-g70f8415btrueOK43554358mirror.1vm0720.2.0-712-g70f8415bfalseOKtest2d4a20c9-f6e9-4ce8-9d64-84febc265629up+replayingreplaying, {"bytes_per_second":3850258.83,"entries_behind_primary":2380,"entries_per_second":239.3,"non_primary_position":{"entry_tid":5267,"object_number":11,"tag_tid":5},"primary_position":{"entry_tid":7647,"object_number":15,"tag_tid":5},"seconds_until_synced":9}42944297mirror.0vm072026-03-24 02:16:51cluster26c644ee3-7db1-4d97-9283-48b1f8a82096up+stoppedlocal image is primary2026-03-24 02:17:04' 2026-03-24T02:17:16.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update state 2026-03-24T02:17:16.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-24T02:17:16.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:16:51' 2026-03-24T02:17:16.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-24T02:17:16.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=up+replaying 2026-03-24T02:17:16.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep up+replaying 2026-03-24T02:17:16.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:17:16.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:17:16.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:16.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:16.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap3 2026-03-24T02:17:16.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap3 2026-03-24T02:17:16.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:17:16.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:16.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:16.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap3 2026-03-24T02:17:16.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:17:31.068 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:17:31.066+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f9110 2026-03-24T02:17:31.068 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:17:31.066+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f91e0 2026-03-24T02:17:31.068 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:17:31.066+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f92b0 2026-03-24T02:17:31.068 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:17:31.066+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f9d40 2026-03-24T02:17:34.097 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.47892701414693384) > weight (0.25) 2026-03-24T02:17:34.097 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-03-24T02:17:34.097 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-03-24T02:17:34.098 INFO:tasks.rbd_mirror_thrash:waiting for 8.852051565307718 secs before reviving daemons 2026-03-24T02:17:34.099 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:17:34.093+0000 7f66d2c47640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 26426) UID: 1000 2026-03-24T02:17:42.950 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-03-24T02:17:42.950 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:17:42.950 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-03-24T02:17:42.950 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-03-24T02:17:42.950 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-03-24T02:17:42.951 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-03-24T02:17:42.952 INFO:tasks.rbd_mirror_thrash:waiting for 100.3832314969958 secs before thrashing 2026-03-24T02:17:49.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-24T02:17:49.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:49.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:49.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:49.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-24T02:17:49.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:49.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:49.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:49.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:17:49.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:17:49.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:49.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:17:50.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-24T02:17:50.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:50.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:50.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:50.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:17:50.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:17:50.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:17:50.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-24T02:17:50.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:17:50.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:17:50.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:17:50.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:50.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:17:50.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:17:50.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:50.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:17:50.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:50.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:17:50.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:50.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-24T02:17:50.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:17:50.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:17:50.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:50.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:50.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:50.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:17:50.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:50.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:50.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:50.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:50.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:17:50.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:17:50.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:50.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:50.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:50.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:17:50.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:17:50.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:17:50.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:50.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:17:50.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:17:50.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:17:50.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:17:50.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:50.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:50.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:17:50.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:17:50.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:17:50.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:17:50.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:50.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:50.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:50.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:50.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-24T02:17:50.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:17:50.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:17:50.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:17:50.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:50.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:50.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:50.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:50.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:50.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:50.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:50.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:50.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:50.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:50.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:50.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:50.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:50.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:50.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:50.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:50.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:50.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:50.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:50.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:50.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:50.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:50.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:50.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:50.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:50.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:50.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:50.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:50.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:50.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:50.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:50.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:50.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:50.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:50.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:50.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:50.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:50.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:50.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:50.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:50.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:50.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:50.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:50.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:50.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:50.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:50.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:50.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:50.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:50.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:50.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:50.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:50.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:50.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:50.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:50.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=10539], [object_number=30, tag_tid=8, entry_tid=10538], [object_number=29, tag_tid=8, entry_tid=10537], [object_number=28, tag_tid=8, entry_tid=10536]]], state=connected] 2026-03-24T02:17:50.731 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:50.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:50.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:50.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:50.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:50.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=10539], [object_number=30, tag_tid=8, entry_tid=10538], [object_number=29, tag_tid=8, entry_tid=10537], [object_number=28, tag_tid=8, entry_tid=10536]]], state=connected] 2026-03-24T02:17:50.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:50.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=31, tag_tid=8, entry_tid=10539' 2026-03-24T02:17:50.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=31, tag_tid=8, entry_tid=10539' 2026-03-24T02:17:50.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=10539' '!=' '' 2026-03-24T02:17:50.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:50.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=10539' = '' 2026-03-24T02:17:50.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=31, tag_tid=8, entry_tid=10539' 2026-03-24T02:17:50.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:50.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:50.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:50.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:50.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=10539' 2026-03-24T02:17:50.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:50.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:50.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:50.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:50.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:50.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:50.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:50.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=10539' 2026-03-24T02:17:50.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:50.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:50.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=10539 2026-03-24T02:17:50.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 10539 2026-03-24T02:17:50.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:50.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:50.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:50.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:50.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:50.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:50.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:50.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:50.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:50.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:50.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:50.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:50.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:50.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:50.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:50.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:50.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:50.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:50.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:50.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:50.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:50.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:50.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:50.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:50.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:50.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:50.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:50.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:50.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:51.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:51.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:51.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:51.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:51.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:51.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:51.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:51.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:51.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:51.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:51.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:51.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:51.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:51.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:51.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:51.095 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:51.095 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:51.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:51.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:51.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:51.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:51.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:51.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=10634], [object_number=29, tag_tid=8, entry_tid=10633], [object_number=28, tag_tid=8, entry_tid=10632], [object_number=31, tag_tid=8, entry_tid=10631]]], state=connected] 2026-03-24T02:17:51.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:51.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:51.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:51.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:51.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:51.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:51.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:51.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:51.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:51.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:51.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:51.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:51.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:51.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:51.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:51.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:51.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:51.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:51.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=10634], [object_number=29, tag_tid=8, entry_tid=10633], [object_number=28, tag_tid=8, entry_tid=10632], [object_number=31, tag_tid=8, entry_tid=10631]]], state=connected] 2026-03-24T02:17:51.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=30, tag_tid=8, entry_tid=10634' 2026-03-24T02:17:51.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=30, tag_tid=8, entry_tid=10634' 2026-03-24T02:17:51.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=10634' '!=' 'object_number=31, tag_tid=8, entry_tid=10539' 2026-03-24T02:17:51.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:51.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=10634' = 'object_number=31, tag_tid=8, entry_tid=10539' 2026-03-24T02:17:51.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=30, tag_tid=8, entry_tid=10634' 2026-03-24T02:17:51.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:51.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:51.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:51.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:51.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=10634' 2026-03-24T02:17:51.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:51.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:51.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:51.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:51.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:51.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:51.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:51.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=10634' 2026-03-24T02:17:51.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:51.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:51.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=10634 2026-03-24T02:17:51.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 10634 2026-03-24T02:17:51.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:51.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:51.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:51.390 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:51.390 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:51.390 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:51.390 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:51.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:51.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:51.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:51.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:51.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:51.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:51.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:51.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:51.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:51.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:51.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:51.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:51.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:51.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:51.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:51.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:51.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:51.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:51.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:51.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:51.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:51.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:51.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:51.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:51.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:51.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:51.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:51.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:51.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:51.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:51.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:51.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:51.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:51.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:51.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:51.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:51.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:51.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:51.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:51.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:51.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:51.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:51.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:51.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:51.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:51.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:51.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:51.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:51.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:51.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=10729], [object_number=28, tag_tid=8, entry_tid=10728], [object_number=31, tag_tid=8, entry_tid=10727], [object_number=30, tag_tid=8, entry_tid=10726]]], state=connected] 2026-03-24T02:17:51.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:51.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:51.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:51.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:51.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=10729], [object_number=28, tag_tid=8, entry_tid=10728], [object_number=31, tag_tid=8, entry_tid=10727], [object_number=30, tag_tid=8, entry_tid=10726]]], state=connected] 2026-03-24T02:17:51.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=29, tag_tid=8, entry_tid=10729' 2026-03-24T02:17:51.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=29, tag_tid=8, entry_tid=10729' 2026-03-24T02:17:51.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=10729' '!=' 'object_number=30, tag_tid=8, entry_tid=10634' 2026-03-24T02:17:51.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:51.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=10729' = 'object_number=30, tag_tid=8, entry_tid=10634' 2026-03-24T02:17:51.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=29, tag_tid=8, entry_tid=10729' 2026-03-24T02:17:51.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:51.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:51.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:51.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:51.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=10729' 2026-03-24T02:17:51.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:51.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:51.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:51.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:51.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:51.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:51.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:51.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=10729' 2026-03-24T02:17:51.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:51.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:51.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=10729 2026-03-24T02:17:51.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 10729 2026-03-24T02:17:51.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:51.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:51.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:51.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:51.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:51.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:51.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:51.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:51.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:51.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:51.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:51.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:51.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:51.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:51.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:51.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:51.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:51.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:51.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:51.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:51.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:51.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:51.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:51.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:51.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:51.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:51.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:51.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:51.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:51.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:51.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:51.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:51.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:51.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:51.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:51.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:51.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:51.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:51.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:51.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:51.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:51.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:51.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:51.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:51.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:51.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=10779], [object_number=30, tag_tid=8, entry_tid=10778], [object_number=29, tag_tid=8, entry_tid=10777], [object_number=28, tag_tid=8, entry_tid=10776]]], state=connected] 2026-03-24T02:17:51.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:51.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:51.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:51.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:51.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:51.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=10779], [object_number=30, tag_tid=8, entry_tid=10778], [object_number=29, tag_tid=8, entry_tid=10777], [object_number=28, tag_tid=8, entry_tid=10776]]], state=connected] 2026-03-24T02:17:51.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:51.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=31, tag_tid=8, entry_tid=10779' 2026-03-24T02:17:51.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=31, tag_tid=8, entry_tid=10779' 2026-03-24T02:17:51.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=10779' '!=' 'object_number=29, tag_tid=8, entry_tid=10729' 2026-03-24T02:17:51.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:51.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=10779' = 'object_number=29, tag_tid=8, entry_tid=10729' 2026-03-24T02:17:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=31, tag_tid=8, entry_tid=10779' 2026-03-24T02:17:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:51.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:51.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=10779' 2026-03-24T02:17:51.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:51.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:52.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:52.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:52.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:52.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:52.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:52.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=10779' 2026-03-24T02:17:52.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:52.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:52.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=10779 2026-03-24T02:17:52.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 10779 2026-03-24T02:17:52.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:52.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:52.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:52.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:52.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:52.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:52.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:52.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:52.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:52.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:52.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:52.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:52.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:52.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:52.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:52.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:52.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:52.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:52.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:52.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:52.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:52.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:52.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:52.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:52.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:52.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:52.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:52.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:52.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:52.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:52.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:52.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:52.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:52.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:52.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:52.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:52.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:52.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:52.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:52.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:52.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:52.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:52.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:52.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:52.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:52.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:52.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:52.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:52.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:52.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:52.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:52.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:52.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:52.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:52.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:52.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:52.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:52.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=10868], [object_number=31, tag_tid=8, entry_tid=10867], [object_number=30, tag_tid=8, entry_tid=10866], [object_number=29, tag_tid=8, entry_tid=10865]]], state=connected] 2026-03-24T02:17:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:52.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:52.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:52.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:52.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:52.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:52.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:52.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:52.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:52.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:52.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=10868], [object_number=31, tag_tid=8, entry_tid=10867], [object_number=30, tag_tid=8, entry_tid=10866], [object_number=29, tag_tid=8, entry_tid=10865]]], state=connected] 2026-03-24T02:17:52.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:52.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=28, tag_tid=8, entry_tid=10868' 2026-03-24T02:17:52.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=28, tag_tid=8, entry_tid=10868' 2026-03-24T02:17:52.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=10868' '!=' 'object_number=31, tag_tid=8, entry_tid=10779' 2026-03-24T02:17:52.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:52.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=10868' = 'object_number=31, tag_tid=8, entry_tid=10779' 2026-03-24T02:17:52.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=28, tag_tid=8, entry_tid=10868' 2026-03-24T02:17:52.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:52.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:52.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:52.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:52.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=10868' 2026-03-24T02:17:52.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:52.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:52.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:52.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:52.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:52.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:52.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:52.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=10868' 2026-03-24T02:17:52.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:52.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:52.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=10868 2026-03-24T02:17:52.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 10868 2026-03-24T02:17:52.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:52.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:52.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:52.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:52.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:52.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:52.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:52.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:52.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:52.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:52.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:52.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:52.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:52.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:52.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:52.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:52.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:52.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:52.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:52.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:52.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:52.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:52.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:52.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:52.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:52.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:52.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:52.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:52.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:52.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:52.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:52.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:52.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:52.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:52.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:52.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:52.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:52.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:52.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:52.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:52.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:52.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:52.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:52.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:52.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:52.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:52.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:52.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:52.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:52.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:52.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:52.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:52.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:52.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=11018], [object_number=29, tag_tid=8, entry_tid=11017], [object_number=28, tag_tid=8, entry_tid=11016], [object_number=31, tag_tid=8, entry_tid=11015]]], state=connected] 2026-03-24T02:17:52.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:52.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:52.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:52.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:52.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:52.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:52.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:52.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:52.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:52.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:52.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:52.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:52.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:52.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:52.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:52.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:52.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:52.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:52.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:52.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:52.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:52.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=11018], [object_number=29, tag_tid=8, entry_tid=11017], [object_number=28, tag_tid=8, entry_tid=11016], [object_number=31, tag_tid=8, entry_tid=11015]]], state=connected] 2026-03-24T02:17:53.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:53.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=30, tag_tid=8, entry_tid=11018' 2026-03-24T02:17:53.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=30, tag_tid=8, entry_tid=11018' 2026-03-24T02:17:53.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=11018' '!=' 'object_number=28, tag_tid=8, entry_tid=10868' 2026-03-24T02:17:53.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:53.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=11018' = 'object_number=28, tag_tid=8, entry_tid=10868' 2026-03-24T02:17:53.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=30, tag_tid=8, entry_tid=11018' 2026-03-24T02:17:53.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:53.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:53.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:53.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:53.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=11018' 2026-03-24T02:17:53.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:53.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:53.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:53.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:53.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:53.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:53.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:53.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=11018' 2026-03-24T02:17:53.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:53.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:53.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=11018 2026-03-24T02:17:53.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 11018 2026-03-24T02:17:53.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:53.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:53.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:53.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:53.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:53.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:53.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:53.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:53.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:53.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:53.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:53.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:53.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:53.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:53.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:53.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:53.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:53.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:53.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:53.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:53.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:53.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:53.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:53.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:53.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:53.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:53.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:53.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:53.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:53.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:53.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:53.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:53.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:53.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:53.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:53.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:53.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:53.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:53.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:53.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:53.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:53.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:53.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:53.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:53.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=11099], [object_number=30, tag_tid=8, entry_tid=11098], [object_number=29, tag_tid=8, entry_tid=11097], [object_number=28, tag_tid=8, entry_tid=11096]]], state=connected] 2026-03-24T02:17:53.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:53.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:53.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:53.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:53.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:53.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:53.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:53.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:53.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:53.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:53.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:53.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:53.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=11099], [object_number=30, tag_tid=8, entry_tid=11098], [object_number=29, tag_tid=8, entry_tid=11097], [object_number=28, tag_tid=8, entry_tid=11096]]], state=connected] 2026-03-24T02:17:53.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:53.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=31, tag_tid=8, entry_tid=11099' 2026-03-24T02:17:53.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=31, tag_tid=8, entry_tid=11099' 2026-03-24T02:17:53.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=11099' '!=' 'object_number=30, tag_tid=8, entry_tid=11018' 2026-03-24T02:17:53.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:53.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=11099' = 'object_number=30, tag_tid=8, entry_tid=11018' 2026-03-24T02:17:53.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=31, tag_tid=8, entry_tid=11099' 2026-03-24T02:17:53.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:53.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:53.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:53.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:53.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=11099' 2026-03-24T02:17:53.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:53.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:53.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:53.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:53.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:53.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:53.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:53.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=11099' 2026-03-24T02:17:53.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:53.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:53.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=11099 2026-03-24T02:17:53.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 11099 2026-03-24T02:17:53.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:53.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:53.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:53.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:53.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:53.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:53.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:53.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:53.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:53.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:53.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:53.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:53.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:53.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:53.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:53.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:53.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:53.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:53.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:53.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:53.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:53.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:53.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:53.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:53.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:53.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:53.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:53.733 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:53.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:53.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:53.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:53.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:53.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:53.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=11190], [object_number=29, tag_tid=8, entry_tid=11189], [object_number=28, tag_tid=8, entry_tid=11188], [object_number=31, tag_tid=8, entry_tid=11187]]], state=connected] 2026-03-24T02:17:53.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:53.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:53.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:53.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:53.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:53.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:53.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:53.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:53.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=11190], [object_number=29, tag_tid=8, entry_tid=11189], [object_number=28, tag_tid=8, entry_tid=11188], [object_number=31, tag_tid=8, entry_tid=11187]]], state=connected] 2026-03-24T02:17:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:53.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=30, tag_tid=8, entry_tid=11190' 2026-03-24T02:17:53.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=30, tag_tid=8, entry_tid=11190' 2026-03-24T02:17:53.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=11190' '!=' 'object_number=31, tag_tid=8, entry_tid=11099' 2026-03-24T02:17:53.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:53.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=11190' = 'object_number=31, tag_tid=8, entry_tid=11099' 2026-03-24T02:17:53.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=30, tag_tid=8, entry_tid=11190' 2026-03-24T02:17:53.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:53.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:53.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:53.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:53.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=11190' 2026-03-24T02:17:53.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:53.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:53.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:53.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:53.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:53.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:53.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:53.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=11190' 2026-03-24T02:17:53.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:53.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:53.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=11190 2026-03-24T02:17:53.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 11190 2026-03-24T02:17:53.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:53.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:53.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:54.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:54.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:54.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:54.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:54.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:54.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:54.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:54.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:54.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:54.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:54.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:54.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:54.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:54.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:54.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:54.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:54.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:54.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:54.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:54.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:54.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:54.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:54.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:54.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:54.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:54.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:54.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:54.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:54.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:54.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:54.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:54.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:54.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:54.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:54.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:54.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:54.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:54.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:54.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:54.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:54.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=11264], [object_number=31, tag_tid=8, entry_tid=11263], [object_number=30, tag_tid=8, entry_tid=11262], [object_number=29, tag_tid=8, entry_tid=11261]]], state=connected] 2026-03-24T02:17:54.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:54.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:54.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:54.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:54.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:54.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:54.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:54.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:54.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:54.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:54.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:54.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:54.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:54.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:54.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:54.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:54.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=11264], [object_number=31, tag_tid=8, entry_tid=11263], [object_number=30, tag_tid=8, entry_tid=11262], [object_number=29, tag_tid=8, entry_tid=11261]]], state=connected] 2026-03-24T02:17:54.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:54.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=28, tag_tid=8, entry_tid=11264' 2026-03-24T02:17:54.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=28, tag_tid=8, entry_tid=11264' 2026-03-24T02:17:54.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=11264' '!=' 'object_number=30, tag_tid=8, entry_tid=11190' 2026-03-24T02:17:54.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:54.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=11264' = 'object_number=30, tag_tid=8, entry_tid=11190' 2026-03-24T02:17:54.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=28, tag_tid=8, entry_tid=11264' 2026-03-24T02:17:54.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:54.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:54.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:54.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:54.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=11264' 2026-03-24T02:17:54.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:54.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:54.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:54.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:54.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:54.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:54.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:54.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=11264' 2026-03-24T02:17:54.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:54.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:54.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=11264 2026-03-24T02:17:54.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 11264 2026-03-24T02:17:54.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:54.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:54.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:54.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:54.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:54.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:54.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:54.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:54.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:54.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:54.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:54.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:54.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:54.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:54.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:54.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:54.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:54.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:54.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:54.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:54.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:54.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:54.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:54.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:54.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:54.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:54.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:54.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:54.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:54.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:54.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:54.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:54.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:54.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:54.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:54.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:54.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:54.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:54.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:54.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:54.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:54.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:54.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:54.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:54.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:54.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:54.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:54.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:54.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:54.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:54.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:54.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:54.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:54.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:54.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:54.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:54.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:54.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:54.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:54.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:54.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:54.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:54.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:54.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:54.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=11347], [object_number=30, tag_tid=8, entry_tid=11346], [object_number=29, tag_tid=8, entry_tid=11345], [object_number=28, tag_tid=8, entry_tid=11344]]], state=connected] 2026-03-24T02:17:54.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:54.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:54.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:54.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:54.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:54.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:54.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:54.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:54.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=11347], [object_number=30, tag_tid=8, entry_tid=11346], [object_number=29, tag_tid=8, entry_tid=11345], [object_number=28, tag_tid=8, entry_tid=11344]]], state=connected] 2026-03-24T02:17:54.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:54.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=31, tag_tid=8, entry_tid=11347' 2026-03-24T02:17:54.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=31, tag_tid=8, entry_tid=11347' 2026-03-24T02:17:54.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=11347' '!=' 'object_number=28, tag_tid=8, entry_tid=11264' 2026-03-24T02:17:54.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:54.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=11347' = 'object_number=28, tag_tid=8, entry_tid=11264' 2026-03-24T02:17:54.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=31, tag_tid=8, entry_tid=11347' 2026-03-24T02:17:54.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:54.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:54.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:54.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:54.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=11347' 2026-03-24T02:17:54.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:54.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:54.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:54.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:54.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:54.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:54.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:54.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=11347' 2026-03-24T02:17:54.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:54.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:54.832 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=11347 2026-03-24T02:17:54.832 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 11347 2026-03-24T02:17:54.832 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:54.832 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:54.832 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:55.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:55.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:55.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:55.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:55.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:55.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:55.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:55.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:55.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:55.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:55.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:55.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:55.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:55.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:55.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:55.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:55.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:55.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:55.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:55.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:55.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:55.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:55.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:55.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:55.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:55.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:55.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:55.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:55.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:55.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:55.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:55.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:55.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:55.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:55.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:55.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:55.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:55.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:55.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:55.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:55.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=11484], [object_number=31, tag_tid=8, entry_tid=11483], [object_number=30, tag_tid=8, entry_tid=11482], [object_number=29, tag_tid=8, entry_tid=11481]]], state=connected] 2026-03-24T02:17:55.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:55.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:55.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:55.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:55.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:55.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:55.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:55.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:55.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:55.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:55.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:55.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:55.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:55.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:55.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:55.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:55.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:55.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:55.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:55.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:55.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:55.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=11484], [object_number=31, tag_tid=8, entry_tid=11483], [object_number=30, tag_tid=8, entry_tid=11482], [object_number=29, tag_tid=8, entry_tid=11481]]], state=connected] 2026-03-24T02:17:55.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:55.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=28, tag_tid=8, entry_tid=11484' 2026-03-24T02:17:55.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=28, tag_tid=8, entry_tid=11484' 2026-03-24T02:17:55.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=11484' '!=' 'object_number=31, tag_tid=8, entry_tid=11347' 2026-03-24T02:17:55.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:55.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=11484' = 'object_number=31, tag_tid=8, entry_tid=11347' 2026-03-24T02:17:55.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=28, tag_tid=8, entry_tid=11484' 2026-03-24T02:17:55.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:55.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:55.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:55.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:55.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=11484' 2026-03-24T02:17:55.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:55.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:55.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:55.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:55.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:55.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:55.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:55.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=11484' 2026-03-24T02:17:55.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:55.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:55.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=11484 2026-03-24T02:17:55.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 11484 2026-03-24T02:17:55.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:55.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:55.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:55.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:55.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:55.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:55.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:55.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:55.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:55.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:55.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:55.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:55.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:55.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:55.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:55.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:55.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:55.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:55.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:55.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:55.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:55.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:55.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:55.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:55.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:55.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:55.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:55.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:55.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:55.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:55.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:55.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:55.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:55.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:55.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:55.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:55.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:55.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:55.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:55.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:55.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:55.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:55.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:55.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:55.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:55.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:55.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:55.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:55.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:55.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:55.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:55.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:55.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:55.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=11578], [object_number=29, tag_tid=8, entry_tid=11577], [object_number=28, tag_tid=8, entry_tid=11576], [object_number=31, tag_tid=8, entry_tid=11575]]], state=connected] 2026-03-24T02:17:55.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:55.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:55.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:55.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:55.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:55.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:55.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:55.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:55.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:55.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:55.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:55.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:55.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:55.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:55.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:55.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:55.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:55.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:55.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:55.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:55.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:55.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=11578], [object_number=29, tag_tid=8, entry_tid=11577], [object_number=28, tag_tid=8, entry_tid=11576], [object_number=31, tag_tid=8, entry_tid=11575]]], state=connected] 2026-03-24T02:17:55.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:55.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=30, tag_tid=8, entry_tid=11578' 2026-03-24T02:17:55.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=30, tag_tid=8, entry_tid=11578' 2026-03-24T02:17:55.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=11578' '!=' 'object_number=28, tag_tid=8, entry_tid=11484' 2026-03-24T02:17:55.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:55.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=11578' = 'object_number=28, tag_tid=8, entry_tid=11484' 2026-03-24T02:17:55.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=30, tag_tid=8, entry_tid=11578' 2026-03-24T02:17:55.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:55.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:55.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:55.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:55.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=11578' 2026-03-24T02:17:55.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:55.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:55.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:55.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:55.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:55.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:55.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:55.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=11578' 2026-03-24T02:17:55.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:55.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:55.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=11578 2026-03-24T02:17:55.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 11578 2026-03-24T02:17:55.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:55.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:55.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:55.833 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:55.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:55.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:55.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:55.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:55.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:55.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:55.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:55.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:55.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:55.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:55.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:55.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:55.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:55.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:55.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:55.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:55.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:55.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:55.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:55.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:55.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:55.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:55.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:55.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:55.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:55.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:55.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:55.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:55.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:55.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:55.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:55.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:55.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:55.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:55.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:55.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=11668], [object_number=31, tag_tid=8, entry_tid=11667], [object_number=30, tag_tid=8, entry_tid=11666], [object_number=29, tag_tid=8, entry_tid=11665]]], state=connected] 2026-03-24T02:17:55.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:55.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:55.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:55.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:55.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:55.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:55.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:55.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:55.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:55.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:55.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:55.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:55.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:55.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:55.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:55.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:55.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=11668], [object_number=31, tag_tid=8, entry_tid=11667], [object_number=30, tag_tid=8, entry_tid=11666], [object_number=29, tag_tid=8, entry_tid=11665]]], state=connected] 2026-03-24T02:17:56.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:56.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=28, tag_tid=8, entry_tid=11668' 2026-03-24T02:17:56.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=28, tag_tid=8, entry_tid=11668' 2026-03-24T02:17:56.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=11668' '!=' 'object_number=30, tag_tid=8, entry_tid=11578' 2026-03-24T02:17:56.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:56.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=11668' = 'object_number=30, tag_tid=8, entry_tid=11578' 2026-03-24T02:17:56.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=28, tag_tid=8, entry_tid=11668' 2026-03-24T02:17:56.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:56.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:56.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:56.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:56.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=11668' 2026-03-24T02:17:56.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:56.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:56.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:56.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:56.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:56.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:56.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:56.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=11668' 2026-03-24T02:17:56.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:56.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:56.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=11668 2026-03-24T02:17:56.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 11668 2026-03-24T02:17:56.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:56.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:56.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:56.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:56.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:56.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:56.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:56.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:56.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:56.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:56.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:56.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:56.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:56.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:56.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:56.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:56.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:56.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:56.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:56.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:56.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:56.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:56.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:56.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:56.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:56.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:56.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:56.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:56.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:56.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:56.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:56.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:56.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:56.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:56.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:56.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:56.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:56.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:56.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:56.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:56.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:56.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:56.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:56.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:56.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:56.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:56.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:56.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:56.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:56.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:56.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:56.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:56.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:56.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:56.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:56.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:56.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:56.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:56.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:56.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:56.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:56.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:56.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:56.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:56.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:56.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:56.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:56.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:56.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:56.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:56.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:56.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:56.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:56.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:56.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:56.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:56.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=11757], [object_number=28, tag_tid=8, entry_tid=11756], [object_number=31, tag_tid=8, entry_tid=11755], [object_number=30, tag_tid=8, entry_tid=11754]]], state=connected] 2026-03-24T02:17:56.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:56.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:56.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:56.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:56.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:56.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:56.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:56.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:56.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:56.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:56.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:56.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:56.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:56.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:56.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:56.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:56.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:56.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:56.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:56.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:56.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:56.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=11757], [object_number=28, tag_tid=8, entry_tid=11756], [object_number=31, tag_tid=8, entry_tid=11755], [object_number=30, tag_tid=8, entry_tid=11754]]], state=connected] 2026-03-24T02:17:56.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:56.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=29, tag_tid=8, entry_tid=11757' 2026-03-24T02:17:56.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=29, tag_tid=8, entry_tid=11757' 2026-03-24T02:17:56.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=11757' '!=' 'object_number=28, tag_tid=8, entry_tid=11668' 2026-03-24T02:17:56.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:56.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=11757' = 'object_number=28, tag_tid=8, entry_tid=11668' 2026-03-24T02:17:56.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=29, tag_tid=8, entry_tid=11757' 2026-03-24T02:17:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:56.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:56.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=11757' 2026-03-24T02:17:56.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:56.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:56.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:56.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:56.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:56.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:56.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:56.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=11757' 2026-03-24T02:17:56.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:56.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:56.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=11757 2026-03-24T02:17:56.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 11757 2026-03-24T02:17:56.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:56.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:56.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:56.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:56.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:56.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:56.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:56.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:56.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:56.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:56.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:56.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:56.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:56.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:56.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:56.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:56.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:56.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:56.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:56.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:56.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:56.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:56.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:56.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:56.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:56.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:56.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:56.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:56.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:56.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:56.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:56.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:56.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:56.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:56.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:56.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:56.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:56.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:56.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:56.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:56.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:56.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:56.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:56.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:56.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:56.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:56.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:56.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:56.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:56.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:56.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:56.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:56.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:56.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:56.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:56.753 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:56.753 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:56.753 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:56.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:56.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:56.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:56.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:56.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=11848], [object_number=31, tag_tid=8, entry_tid=11847], [object_number=30, tag_tid=8, entry_tid=11846], [object_number=29, tag_tid=8, entry_tid=11845]]], state=connected] 2026-03-24T02:17:56.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:56.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:56.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:56.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:56.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:56.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:56.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:56.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:56.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:56.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:56.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:56.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=11848], [object_number=31, tag_tid=8, entry_tid=11847], [object_number=30, tag_tid=8, entry_tid=11846], [object_number=29, tag_tid=8, entry_tid=11845]]], state=connected] 2026-03-24T02:17:56.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:56.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=28, tag_tid=8, entry_tid=11848' 2026-03-24T02:17:56.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=28, tag_tid=8, entry_tid=11848' 2026-03-24T02:17:56.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=11848' '!=' 'object_number=29, tag_tid=8, entry_tid=11757' 2026-03-24T02:17:56.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:56.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=11848' = 'object_number=29, tag_tid=8, entry_tid=11757' 2026-03-24T02:17:56.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=28, tag_tid=8, entry_tid=11848' 2026-03-24T02:17:56.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:56.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:56.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:56.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:56.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=11848' 2026-03-24T02:17:56.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:56.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:56.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:56.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:56.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:56.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:56.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:56.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=11848' 2026-03-24T02:17:56.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:56.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:56.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=11848 2026-03-24T02:17:56.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 11848 2026-03-24T02:17:56.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:56.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:56.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:57.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:57.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:57.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:57.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:57.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:57.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:57.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:57.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:57.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:57.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:57.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:57.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:57.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:57.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:57.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:57.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:57.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:57.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:57.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:57.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:57.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:57.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:57.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:57.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:57.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:57.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:57.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:57.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:57.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:57.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:57.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:57.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:57.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:57.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:57.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:57.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:57.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:57.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:57.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:57.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:57.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:57.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:57.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:57.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:57.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:57.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:57.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:57.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:57.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:57.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:57.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:57.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:57.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:57.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:57.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:57.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:57.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:57.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:57.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=11955], [object_number=30, tag_tid=8, entry_tid=11954], [object_number=29, tag_tid=8, entry_tid=11953], [object_number=28, tag_tid=8, entry_tid=11952]]], state=connected] 2026-03-24T02:17:57.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:57.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:57.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:57.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:57.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:57.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:57.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:57.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:57.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:57.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:57.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:57.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:57.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:57.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:57.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:57.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:57.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:57.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:57.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:57.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:57.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:57.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=11955], [object_number=30, tag_tid=8, entry_tid=11954], [object_number=29, tag_tid=8, entry_tid=11953], [object_number=28, tag_tid=8, entry_tid=11952]]], state=connected] 2026-03-24T02:17:57.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:57.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=31, tag_tid=8, entry_tid=11955' 2026-03-24T02:17:57.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=31, tag_tid=8, entry_tid=11955' 2026-03-24T02:17:57.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=11955' '!=' 'object_number=28, tag_tid=8, entry_tid=11848' 2026-03-24T02:17:57.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:57.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=11955' = 'object_number=28, tag_tid=8, entry_tid=11848' 2026-03-24T02:17:57.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=31, tag_tid=8, entry_tid=11955' 2026-03-24T02:17:57.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:57.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:57.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:57.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=11955' 2026-03-24T02:17:57.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:57.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:57.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:57.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:57.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:57.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:57.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:57.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=11955' 2026-03-24T02:17:57.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:57.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:57.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=11955 2026-03-24T02:17:57.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 11955 2026-03-24T02:17:57.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:57.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:57.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:57.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:57.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:57.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:57.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:57.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:57.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:57.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:57.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:57.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:57.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:57.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:57.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:57.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:57.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:57.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:57.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:57.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:57.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:57.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:57.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:57.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:57.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:57.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:57.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:57.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:57.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:57.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:57.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:57.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:57.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:57.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:57.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:57.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:57.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:57.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:57.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:57.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:57.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:57.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:57.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:57.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:57.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:57.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:57.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:57.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:57.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:57.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:57.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:57.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:57.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:57.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:57.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:57.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:57.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=12050], [object_number=29, tag_tid=8, entry_tid=12049], [object_number=28, tag_tid=8, entry_tid=12048], [object_number=31, tag_tid=8, entry_tid=12047]]], state=connected] 2026-03-24T02:17:57.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:57.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:57.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:57.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:57.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:57.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=12050], [object_number=29, tag_tid=8, entry_tid=12049], [object_number=28, tag_tid=8, entry_tid=12048], [object_number=31, tag_tid=8, entry_tid=12047]]], state=connected] 2026-03-24T02:17:57.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:57.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=30, tag_tid=8, entry_tid=12050' 2026-03-24T02:17:57.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=30, tag_tid=8, entry_tid=12050' 2026-03-24T02:17:57.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=12050' '!=' 'object_number=31, tag_tid=8, entry_tid=11955' 2026-03-24T02:17:57.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:57.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=12050' = 'object_number=31, tag_tid=8, entry_tid=11955' 2026-03-24T02:17:57.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=30, tag_tid=8, entry_tid=12050' 2026-03-24T02:17:57.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:57.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:57.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:57.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:57.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=12050' 2026-03-24T02:17:57.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:57.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:57.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:57.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:57.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:57.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:57.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:57.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=12050' 2026-03-24T02:17:57.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:57.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:57.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=12050 2026-03-24T02:17:57.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 12050 2026-03-24T02:17:57.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:57.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:57.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:57.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:57.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:57.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:57.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:57.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:57.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:57.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:57.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:57.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:57.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:57.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:57.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:57.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:57.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:57.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:57.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:57.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:57.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:57.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:57.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:57.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:57.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:57.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:57.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:57.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:57.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:57.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:57.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:57.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:57.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:57.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:57.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:57.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:57.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:57.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:57.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:57.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:57.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:57.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:57.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:57.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:57.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:57.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:57.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:57.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:57.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:57.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:57.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:57.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:57.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:57.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:57.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:57.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:57.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:57.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:57.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:57.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:57.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:57.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:57.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:57.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:57.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:57.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:57.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:57.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:57.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:57.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:57.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:57.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:58.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:58.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:58.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:58.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:58.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=12147], [object_number=30, tag_tid=8, entry_tid=12146], [object_number=29, tag_tid=8, entry_tid=12145], [object_number=28, tag_tid=8, entry_tid=12144]]], state=connected] 2026-03-24T02:17:58.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:58.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:58.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:58.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:58.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:58.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:58.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:58.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:58.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:58.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:58.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:58.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:58.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:58.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:58.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:58.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:58.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:58.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:58.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:58.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:58.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:58.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=12147], [object_number=30, tag_tid=8, entry_tid=12146], [object_number=29, tag_tid=8, entry_tid=12145], [object_number=28, tag_tid=8, entry_tid=12144]]], state=connected] 2026-03-24T02:17:58.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:58.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=31, tag_tid=8, entry_tid=12147' 2026-03-24T02:17:58.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=31, tag_tid=8, entry_tid=12147' 2026-03-24T02:17:58.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=12147' '!=' 'object_number=30, tag_tid=8, entry_tid=12050' 2026-03-24T02:17:58.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:58.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=12147' = 'object_number=30, tag_tid=8, entry_tid=12050' 2026-03-24T02:17:58.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=31, tag_tid=8, entry_tid=12147' 2026-03-24T02:17:58.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:58.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:58.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:58.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=12147' 2026-03-24T02:17:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=12147' 2026-03-24T02:17:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=12147 2026-03-24T02:17:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 12147 2026-03-24T02:17:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:58.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:58.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:58.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:58.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:58.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:58.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:58.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:58.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:58.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:58.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:58.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:58.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:58.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:58.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:58.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:58.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:58.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:58.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:58.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:58.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:58.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:58.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:58.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:58.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:58.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:58.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:58.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:58.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:58.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:58.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:58.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:58.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:58.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:58.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:58.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:58.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:58.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:58.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:58.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:58.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:58.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=12225], [object_number=28, tag_tid=8, entry_tid=12224], [object_number=31, tag_tid=8, entry_tid=12223], [object_number=30, tag_tid=8, entry_tid=12222]]], state=connected] 2026-03-24T02:17:58.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:58.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:58.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:58.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:58.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:58.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:58.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:58.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:58.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:58.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:58.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:58.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:58.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:58.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:58.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:58.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:58.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:58.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:58.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:58.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:58.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:58.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=12225], [object_number=28, tag_tid=8, entry_tid=12224], [object_number=31, tag_tid=8, entry_tid=12223], [object_number=30, tag_tid=8, entry_tid=12222]]], state=connected] 2026-03-24T02:17:58.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=29, tag_tid=8, entry_tid=12225' 2026-03-24T02:17:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=29, tag_tid=8, entry_tid=12225' 2026-03-24T02:17:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=12225' '!=' 'object_number=31, tag_tid=8, entry_tid=12147' 2026-03-24T02:17:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=12225' = 'object_number=31, tag_tid=8, entry_tid=12147' 2026-03-24T02:17:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=29, tag_tid=8, entry_tid=12225' 2026-03-24T02:17:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:58.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=12225' 2026-03-24T02:17:58.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:58.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:58.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:58.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:58.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:58.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:58.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:58.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=12225' 2026-03-24T02:17:58.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:58.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:58.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=12225 2026-03-24T02:17:58.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 12225 2026-03-24T02:17:58.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:58.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:58.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:58.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:58.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:58.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:58.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:58.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:58.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:58.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:58.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:58.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:58.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:58.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:58.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:58.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:58.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:58.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:58.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:58.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:58.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:58.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:58.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:58.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:58.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:58.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:58.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:58.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:58.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:58.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:58.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:58.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:58.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:58.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:58.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:58.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:58.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:58.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:58.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:58.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=12310], [object_number=29, tag_tid=8, entry_tid=12309], [object_number=28, tag_tid=8, entry_tid=12308], [object_number=31, tag_tid=8, entry_tid=12307]]], state=connected] 2026-03-24T02:17:58.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:58.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:58.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:58.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:58.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:58.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:58.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:58.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:58.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:58.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:58.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:58.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:58.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:58.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:58.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:58.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:58.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:58.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:58.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:58.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:58.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:58.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=12310], [object_number=29, tag_tid=8, entry_tid=12309], [object_number=28, tag_tid=8, entry_tid=12308], [object_number=31, tag_tid=8, entry_tid=12307]]], state=connected] 2026-03-24T02:17:58.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:58.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=30, tag_tid=8, entry_tid=12310' 2026-03-24T02:17:58.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=30, tag_tid=8, entry_tid=12310' 2026-03-24T02:17:58.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=12310' '!=' 'object_number=29, tag_tid=8, entry_tid=12225' 2026-03-24T02:17:58.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:58.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=12310' = 'object_number=29, tag_tid=8, entry_tid=12225' 2026-03-24T02:17:58.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=30, tag_tid=8, entry_tid=12310' 2026-03-24T02:17:58.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:58.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:58.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:58.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:58.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=12310' 2026-03-24T02:17:58.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:58.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:58.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:58.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:58.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:58.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:58.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:58.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=12310' 2026-03-24T02:17:58.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:58.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:58.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=12310 2026-03-24T02:17:58.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 12310 2026-03-24T02:17:58.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:58.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:58.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:59.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:59.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:59.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:59.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:59.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:59.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:59.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:59.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:59.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:59.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:59.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:59.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:59.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:59.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:59.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:59.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:59.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:59.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:59.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:59.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:59.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:59.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:59.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:59.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:59.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:59.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:59.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:59.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:59.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:59.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:59.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:59.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:59.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:59.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:59.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:59.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:59.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:59.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:59.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:59.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:59.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=12399], [object_number=30, tag_tid=8, entry_tid=12398], [object_number=29, tag_tid=8, entry_tid=12397], [object_number=28, tag_tid=8, entry_tid=12396]]], state=connected] 2026-03-24T02:17:59.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:59.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:59.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:59.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:59.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:59.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:59.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:59.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:59.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:59.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:59.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:59.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:59.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:59.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:59.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:59.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:59.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:59.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:59.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:59.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:59.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:59.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=12399], [object_number=30, tag_tid=8, entry_tid=12398], [object_number=29, tag_tid=8, entry_tid=12397], [object_number=28, tag_tid=8, entry_tid=12396]]], state=connected] 2026-03-24T02:17:59.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:59.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=31, tag_tid=8, entry_tid=12399' 2026-03-24T02:17:59.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=31, tag_tid=8, entry_tid=12399' 2026-03-24T02:17:59.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=12399' '!=' 'object_number=30, tag_tid=8, entry_tid=12310' 2026-03-24T02:17:59.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:59.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=12399' = 'object_number=30, tag_tid=8, entry_tid=12310' 2026-03-24T02:17:59.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=31, tag_tid=8, entry_tid=12399' 2026-03-24T02:17:59.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:59.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:59.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:59.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:59.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=12399' 2026-03-24T02:17:59.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:59.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:59.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:59.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:59.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:59.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:59.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:59.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=12399' 2026-03-24T02:17:59.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:59.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=12399 2026-03-24T02:17:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 12399 2026-03-24T02:17:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:17:59.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:17:59.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:59.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:17:59.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:17:59.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:17:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:17:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:17:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:17:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:17:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:17:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:17:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:17:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:17:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:17:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:17:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:17:59.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:17:59.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:17:59.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:17:59.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:17:59.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:17:59.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:17:59.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:17:59.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:17:59.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:17:59.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:17:59.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:17:59.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:59.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:17:59.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:59.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:17:59.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:59.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:17:59.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:17:59.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:17:59.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:17:59.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:17:59.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:17:59.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:17:59.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:17:59.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:17:59.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:17:59.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:17:59.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:17:59.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:17:59.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:17:59.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:17:59.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:17:59.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:59.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:59.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:17:59.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:17:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:17:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:17:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:17:59.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:59.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:59.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:59.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:17:59.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:59.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:59.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:59.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:59.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:59.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:59.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:59.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:59.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:59.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:59.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=12457], [object_number=28, tag_tid=8, entry_tid=12456], [object_number=31, tag_tid=8, entry_tid=12455], [object_number=30, tag_tid=8, entry_tid=12454]]], state=connected] 2026-03-24T02:17:59.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:59.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:59.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:17:59.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:59.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:59.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:59.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:17:59.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:17:59.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:17:59.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:17:59.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:17:59.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:17:59.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:17:59.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:17:59.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:59.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:17:59.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:59.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:17:59.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:17:59.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:17:59.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:17:59.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=12457], [object_number=28, tag_tid=8, entry_tid=12456], [object_number=31, tag_tid=8, entry_tid=12455], [object_number=30, tag_tid=8, entry_tid=12454]]], state=connected] 2026-03-24T02:17:59.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:17:59.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=29, tag_tid=8, entry_tid=12457' 2026-03-24T02:17:59.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=29, tag_tid=8, entry_tid=12457' 2026-03-24T02:17:59.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=12457' '!=' 'object_number=31, tag_tid=8, entry_tid=12399' 2026-03-24T02:17:59.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:17:59.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=12457' = 'object_number=31, tag_tid=8, entry_tid=12399' 2026-03-24T02:17:59.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=29, tag_tid=8, entry_tid=12457' 2026-03-24T02:17:59.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:59.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:59.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:59.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:17:59.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=12457' 2026-03-24T02:17:59.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:17:59.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:59.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:17:59.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:17:59.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:59.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:59.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:17:59.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=12457' 2026-03-24T02:17:59.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:17:59.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:17:59.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=12457 2026-03-24T02:17:59.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 12457 2026-03-24T02:17:59.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:17:59.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:17:59.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:00.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:00.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:00.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:00.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:00.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:00.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:00.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:00.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:00.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:00.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:00.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:00.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:00.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:00.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:00.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:00.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:00.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:00.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:00.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:00.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:00.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:00.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:00.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:00.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:00.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:00.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:00.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:00.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:00.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:00.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:00.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=12540], [object_number=31, tag_tid=8, entry_tid=12539], [object_number=30, tag_tid=8, entry_tid=12538], [object_number=29, tag_tid=8, entry_tid=12537]]], state=connected] 2026-03-24T02:18:00.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:00.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:00.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:00.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:00.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:00.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:00.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:00.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:00.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:00.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:00.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:00.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:00.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:00.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:00.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:00.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:00.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=12540], [object_number=31, tag_tid=8, entry_tid=12539], [object_number=30, tag_tid=8, entry_tid=12538], [object_number=29, tag_tid=8, entry_tid=12537]]], state=connected] 2026-03-24T02:18:00.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:00.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=28, tag_tid=8, entry_tid=12540' 2026-03-24T02:18:00.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=28, tag_tid=8, entry_tid=12540' 2026-03-24T02:18:00.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=12540' '!=' 'object_number=29, tag_tid=8, entry_tid=12457' 2026-03-24T02:18:00.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:00.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=12540' = 'object_number=29, tag_tid=8, entry_tid=12457' 2026-03-24T02:18:00.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=28, tag_tid=8, entry_tid=12540' 2026-03-24T02:18:00.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:00.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:00.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:00.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:00.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=12540' 2026-03-24T02:18:00.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:00.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:00.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:00.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:00.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:00.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:00.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:00.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=12540' 2026-03-24T02:18:00.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:00.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:00.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=12540 2026-03-24T02:18:00.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 12540 2026-03-24T02:18:00.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:00.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:00.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:00.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:00.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:00.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:00.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:00.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:00.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:00.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:00.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:00.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:00.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:00.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:00.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:00.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:00.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:00.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:00.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:00.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:00.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:00.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:00.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:00.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:00.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:00.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:00.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:00.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:00.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:00.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:00.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:00.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:00.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:00.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:00.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:00.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:00.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:00.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:00.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:00.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:00.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:00.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:00.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:00.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:00.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:00.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:00.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:00.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:00.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:00.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:00.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=12627], [object_number=30, tag_tid=8, entry_tid=12626], [object_number=29, tag_tid=8, entry_tid=12625], [object_number=28, tag_tid=8, entry_tid=12624]]], state=connected] 2026-03-24T02:18:00.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:00.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:00.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:00.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:00.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:00.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:00.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:00.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:00.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:00.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:00.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:00.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:00.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:00.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:00.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:00.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:00.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:00.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:00.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:00.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:00.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:00.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=12627], [object_number=30, tag_tid=8, entry_tid=12626], [object_number=29, tag_tid=8, entry_tid=12625], [object_number=28, tag_tid=8, entry_tid=12624]]], state=connected] 2026-03-24T02:18:00.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:00.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=31, tag_tid=8, entry_tid=12627' 2026-03-24T02:18:00.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=31, tag_tid=8, entry_tid=12627' 2026-03-24T02:18:00.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=12627' '!=' 'object_number=28, tag_tid=8, entry_tid=12540' 2026-03-24T02:18:00.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:00.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=12627' = 'object_number=28, tag_tid=8, entry_tid=12540' 2026-03-24T02:18:00.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=31, tag_tid=8, entry_tid=12627' 2026-03-24T02:18:00.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:00.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:00.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:00.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:00.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=12627' 2026-03-24T02:18:00.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:00.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:00.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:00.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:00.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:00.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:00.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:00.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=12627' 2026-03-24T02:18:00.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:00.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:00.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=12627 2026-03-24T02:18:00.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 12627 2026-03-24T02:18:00.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:00.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:00.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:00.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:00.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:00.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:00.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:00.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:00.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:00.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:00.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:00.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:00.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:00.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:00.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:00.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:00.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:00.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:00.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:00.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:00.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:00.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:00.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:00.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:00.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:00.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:00.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:00.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:00.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:00.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:00.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:00.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:00.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:00.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:00.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:00.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:00.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:00.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:00.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:00.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:00.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:00.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:00.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:00.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:00.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:00.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:00.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:00.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:00.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:00.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:00.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:00.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:00.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:00.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:00.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:00.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:00.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:00.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:00.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:00.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:00.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:00.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:00.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:00.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:00.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:00.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:00.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:00.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:00.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:00.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:00.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:00.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:00.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:00.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:00.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:00.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:00.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=12709], [object_number=28, tag_tid=8, entry_tid=12708], [object_number=31, tag_tid=8, entry_tid=12707], [object_number=30, tag_tid=8, entry_tid=12706]]], state=connected] 2026-03-24T02:18:00.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:00.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:00.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:00.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:00.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:00.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:00.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:00.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:00.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:00.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:00.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:00.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:00.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:00.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:00.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:00.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:00.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:01.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:01.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:01.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:01.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:01.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=12709], [object_number=28, tag_tid=8, entry_tid=12708], [object_number=31, tag_tid=8, entry_tid=12707], [object_number=30, tag_tid=8, entry_tid=12706]]], state=connected] 2026-03-24T02:18:01.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=29, tag_tid=8, entry_tid=12709' 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=29, tag_tid=8, entry_tid=12709' 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=12709' '!=' 'object_number=31, tag_tid=8, entry_tid=12627' 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=12709' = 'object_number=31, tag_tid=8, entry_tid=12627' 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=29, tag_tid=8, entry_tid=12709' 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=12709' 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=12709' 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=12709 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 12709 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:01.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:01.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:01.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:01.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:01.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:01.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:01.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:01.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:01.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:01.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:01.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:01.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:01.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:01.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:01.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:01.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:01.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:01.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:01.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:01.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:01.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:01.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:01.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:01.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:01.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:01.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:01.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:01.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:01.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:01.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:01.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:01.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=12795], [object_number=30, tag_tid=8, entry_tid=12794], [object_number=29, tag_tid=8, entry_tid=12793], [object_number=28, tag_tid=8, entry_tid=12792]]], state=connected] 2026-03-24T02:18:01.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:01.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:01.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:01.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:01.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:01.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:01.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:01.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:01.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:01.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:01.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:01.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:01.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:01.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:01.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:01.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:01.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:01.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:01.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:01.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:01.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:01.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=12795], [object_number=30, tag_tid=8, entry_tid=12794], [object_number=29, tag_tid=8, entry_tid=12793], [object_number=28, tag_tid=8, entry_tid=12792]]], state=connected] 2026-03-24T02:18:01.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:01.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=31, tag_tid=8, entry_tid=12795' 2026-03-24T02:18:01.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=31, tag_tid=8, entry_tid=12795' 2026-03-24T02:18:01.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=12795' '!=' 'object_number=29, tag_tid=8, entry_tid=12709' 2026-03-24T02:18:01.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:01.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=12795' = 'object_number=29, tag_tid=8, entry_tid=12709' 2026-03-24T02:18:01.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=31, tag_tid=8, entry_tid=12795' 2026-03-24T02:18:01.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:01.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:01.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:01.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:01.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=12795' 2026-03-24T02:18:01.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:01.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:01.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:01.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:01.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:01.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:01.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:01.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=12795' 2026-03-24T02:18:01.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:01.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:01.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=12795 2026-03-24T02:18:01.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 12795 2026-03-24T02:18:01.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:01.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:01.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:01.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:01.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:01.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:01.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:01.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:01.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:01.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:01.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:01.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:01.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:01.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:01.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:01.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:01.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:01.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:01.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:01.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:01.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:01.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:01.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:01.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:01.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:01.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:01.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:01.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:01.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:01.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:01.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:01.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:01.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:01.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:01.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:01.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:01.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:01.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:01.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:01.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:01.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:01.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:01.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:01.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:01.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:01.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:01.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:01.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:01.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:01.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:01.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:01.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:01.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:01.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:01.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:01.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:01.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:01.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:01.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:01.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:01.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:01.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:01.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:01.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:01.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:01.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:01.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:01.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:01.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:01.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:01.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:01.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:01.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:01.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:01.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:01.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:01.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=12872], [object_number=31, tag_tid=8, entry_tid=12871], [object_number=30, tag_tid=8, entry_tid=12870], [object_number=29, tag_tid=8, entry_tid=12869]]], state=connected] 2026-03-24T02:18:01.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:01.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:01.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:01.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:01.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:01.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:01.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:01.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:01.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:01.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:01.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:01.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:01.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:01.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:01.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:01.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:01.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:01.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:01.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:01.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:01.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:01.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=12872], [object_number=31, tag_tid=8, entry_tid=12871], [object_number=30, tag_tid=8, entry_tid=12870], [object_number=29, tag_tid=8, entry_tid=12869]]], state=connected] 2026-03-24T02:18:01.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:01.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=28, tag_tid=8, entry_tid=12872' 2026-03-24T02:18:01.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=28, tag_tid=8, entry_tid=12872' 2026-03-24T02:18:01.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=12872' '!=' 'object_number=31, tag_tid=8, entry_tid=12795' 2026-03-24T02:18:01.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:01.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=12872' = 'object_number=31, tag_tid=8, entry_tid=12795' 2026-03-24T02:18:01.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=28, tag_tid=8, entry_tid=12872' 2026-03-24T02:18:01.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:01.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:01.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:01.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:01.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=12872' 2026-03-24T02:18:01.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:01.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:01.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:01.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:01.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:01.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:01.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:01.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=12872' 2026-03-24T02:18:01.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:01.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:01.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=12872 2026-03-24T02:18:01.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 12872 2026-03-24T02:18:01.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:01.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:01.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:02.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:02.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:02.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:02.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:02.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:02.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:02.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:02.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:02.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:02.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:02.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:02.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:02.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:02.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:02.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:02.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:02.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:02.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:02.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:02.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:02.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:02.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:02.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:02.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:02.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:02.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:02.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:02.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:02.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:02.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:02.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:02.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:02.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:02.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:02.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:02.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:02.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:02.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:02.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:02.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:02.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:02.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:02.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:02.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:02.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:02.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:02.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:02.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:02.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:02.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:02.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:02.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:02.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:02.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:02.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:02.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:02.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:02.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:02.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:02.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:02.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:02.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:02.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:02.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:02.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:02.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:02.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:02.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:02.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:02.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:02.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:02.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=12961], [object_number=28, tag_tid=8, entry_tid=12960], [object_number=31, tag_tid=8, entry_tid=12959], [object_number=30, tag_tid=8, entry_tid=12958]]], state=connected] 2026-03-24T02:18:02.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:02.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:02.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:02.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:02.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:02.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:02.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:02.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:02.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:02.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:02.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:02.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:02.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:02.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:02.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=12961], [object_number=28, tag_tid=8, entry_tid=12960], [object_number=31, tag_tid=8, entry_tid=12959], [object_number=30, tag_tid=8, entry_tid=12958]]], state=connected] 2026-03-24T02:18:02.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=29, tag_tid=8, entry_tid=12961' 2026-03-24T02:18:02.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=29, tag_tid=8, entry_tid=12961' 2026-03-24T02:18:02.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=12961' '!=' 'object_number=28, tag_tid=8, entry_tid=12872' 2026-03-24T02:18:02.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:02.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=12961' = 'object_number=28, tag_tid=8, entry_tid=12872' 2026-03-24T02:18:02.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=29, tag_tid=8, entry_tid=12961' 2026-03-24T02:18:02.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:02.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:02.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:02.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:02.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=12961' 2026-03-24T02:18:02.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:02.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:02.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:02.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:02.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:02.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:02.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:02.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=12961' 2026-03-24T02:18:02.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:02.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:02.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=12961 2026-03-24T02:18:02.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 12961 2026-03-24T02:18:02.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:02.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:02.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:02.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:02.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:02.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:02.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:02.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:02.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:02.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:02.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:02.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:02.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:02.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:02.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:02.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:02.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:02.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:02.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:02.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:02.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:02.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:02.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:02.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:02.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:02.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:02.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:02.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:02.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:02.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:02.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:02.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:02.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:02.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:02.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:02.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:02.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:02.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:02.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:02.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:02.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:02.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:02.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:02.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:02.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:02.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:02.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:02.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:02.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:02.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:02.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:02.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:02.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:02.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:02.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:02.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:02.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:02.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:02.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:02.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:02.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:02.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=13048], [object_number=31, tag_tid=8, entry_tid=13047], [object_number=30, tag_tid=8, entry_tid=13046], [object_number=29, tag_tid=8, entry_tid=13045]]], state=connected] 2026-03-24T02:18:02.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:02.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:02.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:02.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:02.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:02.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:02.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:02.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:02.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:02.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:02.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:02.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:02.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:02.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:02.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:02.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:02.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:02.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:02.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=13048], [object_number=31, tag_tid=8, entry_tid=13047], [object_number=30, tag_tid=8, entry_tid=13046], [object_number=29, tag_tid=8, entry_tid=13045]]], state=connected] 2026-03-24T02:18:02.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=28, tag_tid=8, entry_tid=13048' 2026-03-24T02:18:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=28, tag_tid=8, entry_tid=13048' 2026-03-24T02:18:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=13048' '!=' 'object_number=29, tag_tid=8, entry_tid=12961' 2026-03-24T02:18:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=13048' = 'object_number=29, tag_tid=8, entry_tid=12961' 2026-03-24T02:18:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=28, tag_tid=8, entry_tid=13048' 2026-03-24T02:18:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:02.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:02.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=13048' 2026-03-24T02:18:02.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:02.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:02.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:02.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:02.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:02.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:02.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:02.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=13048' 2026-03-24T02:18:02.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:02.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:02.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=13048 2026-03-24T02:18:02.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 13048 2026-03-24T02:18:02.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:02.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:02.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:02.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:02.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:02.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:02.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:02.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:02.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:02.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:02.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:02.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:02.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:02.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:02.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:02.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:02.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:02.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:02.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:02.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:02.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:02.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:02.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:02.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:02.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:02.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:02.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:02.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:02.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:02.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:02.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:02.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:02.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:02.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:02.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:02.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:02.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:02.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:02.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:02.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:02.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:02.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:02.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:02.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:02.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:02.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:02.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:02.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:02.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:02.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:02.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:02.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:02.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:02.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:02.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:02.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:02.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:02.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:02.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:02.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:02.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:02.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:02.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:02.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:02.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:02.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:02.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:02.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:02.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:02.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:02.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:02.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:02.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:02.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:02.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=13130], [object_number=29, tag_tid=8, entry_tid=13129], [object_number=28, tag_tid=8, entry_tid=13128], [object_number=31, tag_tid=8, entry_tid=13127]]], state=connected] 2026-03-24T02:18:02.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:02.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:02.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:02.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:02.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:02.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:02.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:02.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:02.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:02.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:02.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:02.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:02.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:02.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:02.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:02.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:02.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:02.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:02.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=13130], [object_number=29, tag_tid=8, entry_tid=13129], [object_number=28, tag_tid=8, entry_tid=13128], [object_number=31, tag_tid=8, entry_tid=13127]]], state=connected] 2026-03-24T02:18:02.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:02.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=30, tag_tid=8, entry_tid=13130' 2026-03-24T02:18:02.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=30, tag_tid=8, entry_tid=13130' 2026-03-24T02:18:02.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=13130' '!=' 'object_number=28, tag_tid=8, entry_tid=13048' 2026-03-24T02:18:02.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:02.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=13130' = 'object_number=28, tag_tid=8, entry_tid=13048' 2026-03-24T02:18:02.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=30, tag_tid=8, entry_tid=13130' 2026-03-24T02:18:02.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:02.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:02.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:02.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:02.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=13130' 2026-03-24T02:18:02.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:02.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:03.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:03.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:03.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:03.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:03.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:03.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=13130' 2026-03-24T02:18:03.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:03.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:03.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=13130 2026-03-24T02:18:03.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 13130 2026-03-24T02:18:03.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:03.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:03.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:03.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:03.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:03.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:03.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:03.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:03.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:03.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:03.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:03.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:03.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:03.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:03.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:03.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:03.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:03.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:03.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:03.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:03.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:03.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:03.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:03.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:03.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:03.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:03.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:03.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:03.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:03.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:03.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:03.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:03.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:03.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:03.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:03.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:03.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:03.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:03.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:03.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:03.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:03.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:03.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:03.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:03.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:03.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:03.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:03.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=13209], [object_number=28, tag_tid=8, entry_tid=13208], [object_number=31, tag_tid=8, entry_tid=13207], [object_number=30, tag_tid=8, entry_tid=13206]]], state=connected] 2026-03-24T02:18:03.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:03.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:03.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:03.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:03.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:03.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:03.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:03.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:03.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:03.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:03.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:03.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:03.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:03.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:03.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:03.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:03.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:03.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:03.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:03.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:03.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=13209], [object_number=28, tag_tid=8, entry_tid=13208], [object_number=31, tag_tid=8, entry_tid=13207], [object_number=30, tag_tid=8, entry_tid=13206]]], state=connected] 2026-03-24T02:18:03.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:03.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=29, tag_tid=8, entry_tid=13209' 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=29, tag_tid=8, entry_tid=13209' 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=13209' '!=' 'object_number=30, tag_tid=8, entry_tid=13130' 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=13209' = 'object_number=30, tag_tid=8, entry_tid=13130' 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=29, tag_tid=8, entry_tid=13209' 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=13209' 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=13209' 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:03.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:03.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=13209 2026-03-24T02:18:03.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 13209 2026-03-24T02:18:03.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:03.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:03.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:03.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:03.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:03.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:03.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:03.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:03.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:03.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:03.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:03.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:03.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:03.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:03.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:03.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:03.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:03.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:03.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:03.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:03.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:03.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:03.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:03.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:03.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:03.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:03.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:03.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:03.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:03.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:03.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:03.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:03.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:03.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:03.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:03.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=13290], [object_number=29, tag_tid=8, entry_tid=13289], [object_number=28, tag_tid=8, entry_tid=13288], [object_number=31, tag_tid=8, entry_tid=13287]]], state=connected] 2026-03-24T02:18:03.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:03.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:03.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:03.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:03.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:03.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:03.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:03.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:03.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:03.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:03.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:03.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:03.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:03.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:03.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:03.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:03.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:03.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:03.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:03.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:03.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:03.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=13290], [object_number=29, tag_tid=8, entry_tid=13289], [object_number=28, tag_tid=8, entry_tid=13288], [object_number=31, tag_tid=8, entry_tid=13287]]], state=connected] 2026-03-24T02:18:03.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:03.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=30, tag_tid=8, entry_tid=13290' 2026-03-24T02:18:03.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=30, tag_tid=8, entry_tid=13290' 2026-03-24T02:18:03.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=13290' '!=' 'object_number=29, tag_tid=8, entry_tid=13209' 2026-03-24T02:18:03.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:03.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=13290' = 'object_number=29, tag_tid=8, entry_tid=13209' 2026-03-24T02:18:03.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=30, tag_tid=8, entry_tid=13290' 2026-03-24T02:18:03.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:03.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:03.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:03.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:03.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=13290' 2026-03-24T02:18:03.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:03.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:03.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:03.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:03.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:03.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:03.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:03.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=13290' 2026-03-24T02:18:03.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:03.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:03.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=13290 2026-03-24T02:18:03.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 13290 2026-03-24T02:18:03.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:03.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:03.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:04.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:04.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:04.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:04.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:04.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:04.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:04.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:04.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:04.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:04.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:04.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:04.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:04.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:04.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:04.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:04.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:04.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:04.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:04.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:04.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:04.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:04.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:04.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:04.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:04.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:04.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:04.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:04.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:04.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:04.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:04.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:04.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:04.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:04.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:04.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:04.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:04.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:04.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:04.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:04.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:04.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:04.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:04.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:04.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:04.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:04.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:04.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:04.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:04.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:04.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:04.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:04.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:04.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:04.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:04.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:04.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:04.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:04.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:04.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:04.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:04.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:04.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:04.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:04.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:04.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:04.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:04.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:04.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:04.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:04.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:04.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:04.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:04.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:04.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=13363], [object_number=30, tag_tid=8, entry_tid=13362], [object_number=29, tag_tid=8, entry_tid=13361], [object_number=28, tag_tid=8, entry_tid=13360]]], state=connected] 2026-03-24T02:18:04.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:04.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:04.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:04.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:04.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:04.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:04.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:04.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:04.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:04.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:04.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:04.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:04.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:04.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:04.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:04.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:04.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:04.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:04.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:04.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:04.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:04.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=31, tag_tid=8, entry_tid=13363], [object_number=30, tag_tid=8, entry_tid=13362], [object_number=29, tag_tid=8, entry_tid=13361], [object_number=28, tag_tid=8, entry_tid=13360]]], state=connected] 2026-03-24T02:18:04.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:04.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=31, tag_tid=8, entry_tid=13363' 2026-03-24T02:18:04.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=31, tag_tid=8, entry_tid=13363' 2026-03-24T02:18:04.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=13363' '!=' 'object_number=30, tag_tid=8, entry_tid=13290' 2026-03-24T02:18:04.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:04.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=31, tag_tid=8, entry_tid=13363' = 'object_number=30, tag_tid=8, entry_tid=13290' 2026-03-24T02:18:04.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=31, tag_tid=8, entry_tid=13363' 2026-03-24T02:18:04.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:04.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:04.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:04.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:04.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=13363' 2026-03-24T02:18:04.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:04.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:04.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:04.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:04.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:04.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:04.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:04.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=31, tag_tid=8, entry_tid=13363' 2026-03-24T02:18:04.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:04.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:04.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=13363 2026-03-24T02:18:04.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 13363 2026-03-24T02:18:04.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:04.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:04.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:04.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:04.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:04.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:04.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:04.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:04.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:04.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:04.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:04.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:04.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:04.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:04.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:04.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:04.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:04.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:04.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:04.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:04.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:04.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:04.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:04.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:04.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:04.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:04.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:04.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:04.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:04.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:04.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:04.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:04.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:04.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:04.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:04.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:04.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=13438], [object_number=29, tag_tid=8, entry_tid=13437], [object_number=28, tag_tid=8, entry_tid=13436], [object_number=31, tag_tid=8, entry_tid=13435]]], state=connected] 2026-03-24T02:18:04.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:04.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:04.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:04.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:04.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:04.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:04.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:04.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:04.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:04.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:04.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:04.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:04.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:04.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:04.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:04.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:04.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:04.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:04.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:04.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:04.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:04.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=30, tag_tid=8, entry_tid=13438], [object_number=29, tag_tid=8, entry_tid=13437], [object_number=28, tag_tid=8, entry_tid=13436], [object_number=31, tag_tid=8, entry_tid=13435]]], state=connected] 2026-03-24T02:18:04.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:04.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=30, tag_tid=8, entry_tid=13438' 2026-03-24T02:18:04.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=30, tag_tid=8, entry_tid=13438' 2026-03-24T02:18:04.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=13438' '!=' 'object_number=31, tag_tid=8, entry_tid=13363' 2026-03-24T02:18:04.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:04.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=30, tag_tid=8, entry_tid=13438' = 'object_number=31, tag_tid=8, entry_tid=13363' 2026-03-24T02:18:04.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=30, tag_tid=8, entry_tid=13438' 2026-03-24T02:18:04.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:04.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:04.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:04.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:04.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=13438' 2026-03-24T02:18:04.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:04.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:04.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:04.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:04.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:04.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:04.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:04.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=30, tag_tid=8, entry_tid=13438' 2026-03-24T02:18:04.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:04.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:04.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=13438 2026-03-24T02:18:04.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 13438 2026-03-24T02:18:04.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:04.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:04.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:04.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:04.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:04.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:04.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:04.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:04.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:04.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:04.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:04.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:04.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:04.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:04.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:04.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:04.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:04.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:04.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:04.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:04.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:04.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:04.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:04.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:04.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:04.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:04.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:04.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:04.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:04.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:04.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:04.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:04.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:05.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:05.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:05.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:05.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:05.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:05.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:05.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:05.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:05.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:05.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:05.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:05.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:05.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:05.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:05.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:05.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:05.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:05.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:05.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:05.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:05.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:05.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:05.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:05.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=13489], [object_number=28, tag_tid=8, entry_tid=13488], [object_number=31, tag_tid=8, entry_tid=13487], [object_number=30, tag_tid=8, entry_tid=13486]]], state=connected] 2026-03-24T02:18:05.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:05.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:05.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:05.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:05.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:05.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:05.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:05.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:05.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:05.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:05.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:05.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:05.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:05.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:05.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:05.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:05.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=13489], [object_number=28, tag_tid=8, entry_tid=13488], [object_number=31, tag_tid=8, entry_tid=13487], [object_number=30, tag_tid=8, entry_tid=13486]]], state=connected] 2026-03-24T02:18:05.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:05.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=29, tag_tid=8, entry_tid=13489' 2026-03-24T02:18:05.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=29, tag_tid=8, entry_tid=13489' 2026-03-24T02:18:05.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=13489' '!=' 'object_number=30, tag_tid=8, entry_tid=13438' 2026-03-24T02:18:05.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:05.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=13489' = 'object_number=30, tag_tid=8, entry_tid=13438' 2026-03-24T02:18:05.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=29, tag_tid=8, entry_tid=13489' 2026-03-24T02:18:05.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:05.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:05.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:05.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:05.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=13489' 2026-03-24T02:18:05.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:05.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:05.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:05.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:05.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:05.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:05.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:05.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=13489' 2026-03-24T02:18:05.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:05.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:05.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=13489 2026-03-24T02:18:05.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 13489 2026-03-24T02:18:05.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:05.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:05.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:05.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:05.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:05.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:05.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:05.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:05.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:05.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:05.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:05.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:05.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:05.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:05.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:05.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:05.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:05.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:05.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:05.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:05.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:05.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:05.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:05.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:05.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:05.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:05.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:05.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:05.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:05.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:05.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:05.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:05.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:05.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:05.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:05.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:05.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:05.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:05.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:05.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:05.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:05.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:05.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:05.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:05.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=13569], [object_number=28, tag_tid=8, entry_tid=13568], [object_number=31, tag_tid=8, entry_tid=13567], [object_number=30, tag_tid=8, entry_tid=13566]]], state=connected] 2026-03-24T02:18:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:05.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:05.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:05.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:05.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:05.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:05.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:05.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:05.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:05.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:05.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:05.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:05.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:05.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:05.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:05.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:05.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:05.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:05.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:05.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:05.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:05.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=29, tag_tid=8, entry_tid=13569], [object_number=28, tag_tid=8, entry_tid=13568], [object_number=31, tag_tid=8, entry_tid=13567], [object_number=30, tag_tid=8, entry_tid=13566]]], state=connected] 2026-03-24T02:18:05.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:05.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=29, tag_tid=8, entry_tid=13569' 2026-03-24T02:18:05.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=29, tag_tid=8, entry_tid=13569' 2026-03-24T02:18:05.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=13569' '!=' 'object_number=29, tag_tid=8, entry_tid=13489' 2026-03-24T02:18:05.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:05.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=29, tag_tid=8, entry_tid=13569' = 'object_number=29, tag_tid=8, entry_tid=13489' 2026-03-24T02:18:05.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=29, tag_tid=8, entry_tid=13569' 2026-03-24T02:18:05.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:05.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:05.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:05.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:05.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=13569' 2026-03-24T02:18:05.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:05.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:05.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:05.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:05.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:05.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:05.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:05.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=29, tag_tid=8, entry_tid=13569' 2026-03-24T02:18:05.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:05.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:05.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=13569 2026-03-24T02:18:05.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 13569 2026-03-24T02:18:05.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:05.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:05.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:05.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:05.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:05.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:05.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:05.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:05.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:05.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:05.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:05.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:05.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:05.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:05.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:05.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:05.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:05.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:05.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:05.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:05.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:05.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:05.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:05.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:05.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:05.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:05.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:05.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:05.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:05.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:05.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:05.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:05.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:05.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:05.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:05.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:05.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:05.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:05.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:05.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:05.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:05.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:05.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:05.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:05.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:05.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:05.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:05.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:05.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:05.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:05.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:05.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:05.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:05.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:05.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:05.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:05.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:05.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:05.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:05.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:05.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:05.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:05.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:05.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:05.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:05.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:05.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:05.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:05.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:05.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:05.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:05.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:05.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:05.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:05.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:05.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:05.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=13624], [object_number=31, tag_tid=8, entry_tid=13623], [object_number=30, tag_tid=8, entry_tid=13622], [object_number=29, tag_tid=8, entry_tid=13621]]], state=connected] 2026-03-24T02:18:05.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:05.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:05.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:05.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:05.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:05.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:05.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:05.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:05.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:05.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:05.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:05.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:05.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:05.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:05.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:05.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:05.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:06.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 7 2026-03-24T02:18:06.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:06.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:06.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:06.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=28, tag_tid=8, entry_tid=13624], [object_number=31, tag_tid=8, entry_tid=13623], [object_number=30, tag_tid=8, entry_tid=13622], [object_number=29, tag_tid=8, entry_tid=13621]]], state=connected] 2026-03-24T02:18:06.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:06.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=28, tag_tid=8, entry_tid=13624' 2026-03-24T02:18:06.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=28, tag_tid=8, entry_tid=13624' 2026-03-24T02:18:06.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=13624' '!=' 'object_number=29, tag_tid=8, entry_tid=13569' 2026-03-24T02:18:06.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:06.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=28, tag_tid=8, entry_tid=13624' = 'object_number=29, tag_tid=8, entry_tid=13569' 2026-03-24T02:18:06.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=28, tag_tid=8, entry_tid=13624' 2026-03-24T02:18:06.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:06.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:06.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:06.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:06.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=13624' 2026-03-24T02:18:06.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:06.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:06.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:06.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:06.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:06.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:06.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:06.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=28, tag_tid=8, entry_tid=13624' 2026-03-24T02:18:06.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:06.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:06.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=13624 2026-03-24T02:18:06.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 13624 2026-03-24T02:18:06.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:06.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:06.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:06.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:06.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:06.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:06.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:06.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:06.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:06.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:06.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:06.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:06.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:06.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:06.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:06.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:06.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:06.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:06.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:06.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:06.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:06.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:06.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:06.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:06.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:06.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:06.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:06.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:06.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:06.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:06.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:06.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:06.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:06.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:06.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:06.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:06.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:06.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:06.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:06.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:06.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:06.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:06.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:06.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:06.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:06.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=35, tag_tid=8, entry_tid=13711], [object_number=34, tag_tid=8, entry_tid=13710], [object_number=33, tag_tid=8, entry_tid=13709], [object_number=32, tag_tid=8, entry_tid=13708]]], state=connected] 2026-03-24T02:18:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:06.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:06.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:06.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:06.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:06.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:06.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:06.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:06.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:06.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:06.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:06.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:06.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:06.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:06.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:06.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:06.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:06.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:06.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:06.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:06.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:06.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=35, tag_tid=8, entry_tid=13711], [object_number=34, tag_tid=8, entry_tid=13710], [object_number=33, tag_tid=8, entry_tid=13709], [object_number=32, tag_tid=8, entry_tid=13708]]], state=connected] 2026-03-24T02:18:06.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:06.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=35, tag_tid=8, entry_tid=13711' 2026-03-24T02:18:06.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=35, tag_tid=8, entry_tid=13711' 2026-03-24T02:18:06.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=35, tag_tid=8, entry_tid=13711' '!=' 'object_number=28, tag_tid=8, entry_tid=13624' 2026-03-24T02:18:06.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:06.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=35, tag_tid=8, entry_tid=13711' = 'object_number=28, tag_tid=8, entry_tid=13624' 2026-03-24T02:18:06.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=35, tag_tid=8, entry_tid=13711' 2026-03-24T02:18:06.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:06.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:06.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:06.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:06.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=35, tag_tid=8, entry_tid=13711' 2026-03-24T02:18:06.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:06.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:06.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:06.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:06.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:06.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:06.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:06.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=35, tag_tid=8, entry_tid=13711' 2026-03-24T02:18:06.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:06.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:06.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=13711 2026-03-24T02:18:06.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 13711 2026-03-24T02:18:06.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:06.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:06.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:06.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:06.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:06.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:06.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:06.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:06.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:06.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:06.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:06.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:06.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:06.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:06.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:06.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:06.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:06.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:06.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:06.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:06.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:06.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:06.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:06.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:06.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:06.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:06.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:06.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:06.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:06.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:06.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:06.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:06.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:06.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:06.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:06.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:06.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:06.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:06.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:06.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:06.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:06.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:06.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:06.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:06.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:06.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:06.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:06.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:06.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:06.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:06.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:06.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:06.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:06.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:06.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:06.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:06.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:06.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:06.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:06.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:06.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:06.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:06.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:06.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:06.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:06.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:06.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:06.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:06.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:06.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:06.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:06.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:06.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:06.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:06.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:06.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:06.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=32, tag_tid=8, entry_tid=13796], [object_number=35, tag_tid=8, entry_tid=13795], [object_number=34, tag_tid=8, entry_tid=13794], [object_number=33, tag_tid=8, entry_tid=13793]]], state=connected] 2026-03-24T02:18:06.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:06.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:06.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:06.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:06.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:06.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:06.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:06.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:06.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:06.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:06.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:06.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:06.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:06.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:06.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:06.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:06.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:06.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:06.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:06.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=32, tag_tid=8, entry_tid=13796], [object_number=35, tag_tid=8, entry_tid=13795], [object_number=34, tag_tid=8, entry_tid=13794], [object_number=33, tag_tid=8, entry_tid=13793]]], state=connected] 2026-03-24T02:18:06.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:06.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=32, tag_tid=8, entry_tid=13796' 2026-03-24T02:18:06.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=32, tag_tid=8, entry_tid=13796' 2026-03-24T02:18:06.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=32, tag_tid=8, entry_tid=13796' '!=' 'object_number=35, tag_tid=8, entry_tid=13711' 2026-03-24T02:18:06.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:06.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=32, tag_tid=8, entry_tid=13796' = 'object_number=35, tag_tid=8, entry_tid=13711' 2026-03-24T02:18:06.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=32, tag_tid=8, entry_tid=13796' 2026-03-24T02:18:06.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:06.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:06.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:06.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:06.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=32, tag_tid=8, entry_tid=13796' 2026-03-24T02:18:06.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:06.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:06.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:06.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:06.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:06.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:06.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:06.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=32, tag_tid=8, entry_tid=13796' 2026-03-24T02:18:06.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:06.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:06.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=13796 2026-03-24T02:18:06.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 13796 2026-03-24T02:18:06.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:06.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:06.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:07.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:07.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:07.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:07.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:07.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:07.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:07.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:07.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:07.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:07.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:07.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:07.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:07.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:07.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:07.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:07.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:07.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:07.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:07.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:07.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:07.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:07.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:07.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:07.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:07.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:07.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:07.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:07.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:07.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:07.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:07.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:07.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:07.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:07.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:07.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:07.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:07.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:07.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:07.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:07.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:07.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=34, tag_tid=8, entry_tid=13874], [object_number=33, tag_tid=8, entry_tid=13873], [object_number=32, tag_tid=8, entry_tid=13872], [object_number=35, tag_tid=8, entry_tid=13871]]], state=connected] 2026-03-24T02:18:07.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:07.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:07.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:07.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:07.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:07.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:07.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:07.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:07.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:07.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:07.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:07.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:07.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:07.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:07.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:07.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:07.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:07.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:07.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:07.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:07.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:07.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=34, tag_tid=8, entry_tid=13874], [object_number=33, tag_tid=8, entry_tid=13873], [object_number=32, tag_tid=8, entry_tid=13872], [object_number=35, tag_tid=8, entry_tid=13871]]], state=connected] 2026-03-24T02:18:07.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:07.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=34, tag_tid=8, entry_tid=13874' 2026-03-24T02:18:07.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=34, tag_tid=8, entry_tid=13874' 2026-03-24T02:18:07.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=34, tag_tid=8, entry_tid=13874' '!=' 'object_number=32, tag_tid=8, entry_tid=13796' 2026-03-24T02:18:07.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:07.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=34, tag_tid=8, entry_tid=13874' = 'object_number=32, tag_tid=8, entry_tid=13796' 2026-03-24T02:18:07.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=34, tag_tid=8, entry_tid=13874' 2026-03-24T02:18:07.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:07.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:07.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:07.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:07.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=34, tag_tid=8, entry_tid=13874' 2026-03-24T02:18:07.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:07.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:07.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:07.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:07.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:07.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:07.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:07.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=34, tag_tid=8, entry_tid=13874' 2026-03-24T02:18:07.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:07.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:07.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=13874 2026-03-24T02:18:07.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 13874 2026-03-24T02:18:07.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:07.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:07.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:07.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:07.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:07.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:07.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:07.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:07.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:07.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:07.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:07.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:07.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:07.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:07.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:07.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:07.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:07.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:07.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:07.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:07.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:07.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:07.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:07.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:07.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:07.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:07.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:07.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:07.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:07.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:07.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:07.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:07.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:07.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:07.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:07.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:07.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:07.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:07.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:07.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:07.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:07.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:07.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:07.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:07.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:07.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:07.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:07.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:07.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:07.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:07.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:07.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:07.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:07.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=35, tag_tid=8, entry_tid=13959], [object_number=34, tag_tid=8, entry_tid=13958], [object_number=33, tag_tid=8, entry_tid=13957], [object_number=32, tag_tid=8, entry_tid=13956]]], state=connected] 2026-03-24T02:18:07.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:07.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:07.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:07.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:07.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:07.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:07.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:07.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:07.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:07.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:07.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:07.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:07.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:07.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:07.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:07.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:07.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:07.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:07.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:07.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:07.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:07.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=35, tag_tid=8, entry_tid=13959], [object_number=34, tag_tid=8, entry_tid=13958], [object_number=33, tag_tid=8, entry_tid=13957], [object_number=32, tag_tid=8, entry_tid=13956]]], state=connected] 2026-03-24T02:18:07.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:07.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=35, tag_tid=8, entry_tid=13959' 2026-03-24T02:18:07.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=35, tag_tid=8, entry_tid=13959' 2026-03-24T02:18:07.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=35, tag_tid=8, entry_tid=13959' '!=' 'object_number=34, tag_tid=8, entry_tid=13874' 2026-03-24T02:18:07.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:07.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=35, tag_tid=8, entry_tid=13959' = 'object_number=34, tag_tid=8, entry_tid=13874' 2026-03-24T02:18:07.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=35, tag_tid=8, entry_tid=13959' 2026-03-24T02:18:07.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:07.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:07.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:07.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:07.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=35, tag_tid=8, entry_tid=13959' 2026-03-24T02:18:07.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:07.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:07.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:07.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:07.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:07.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:07.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:07.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=35, tag_tid=8, entry_tid=13959' 2026-03-24T02:18:07.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:07.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:07.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=13959 2026-03-24T02:18:07.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 13959 2026-03-24T02:18:07.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:07.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:07.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:07.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:07.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:07.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:07.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:07.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:07.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:07.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:07.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:07.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:07.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:07.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:07.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:07.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:07.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:07.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:07.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:07.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:07.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:07.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:07.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:07.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:07.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:07.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:07.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:07.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:07.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:07.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:07.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:07.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:07.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:07.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:07.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:07.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:07.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:07.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:07.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:07.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:07.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:07.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:07.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:07.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:07.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:07.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:07.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:07.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:07.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:07.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:07.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:07.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:07.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:07.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:07.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:07.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:07.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:07.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:07.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:07.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:07.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:07.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:07.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:07.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:07.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:07.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:07.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:07.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:07.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:07.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:07.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:07.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:08.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:08.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:08.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:08.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:08.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=34, tag_tid=8, entry_tid=14038], [object_number=33, tag_tid=8, entry_tid=14037], [object_number=32, tag_tid=8, entry_tid=14036], [object_number=35, tag_tid=8, entry_tid=14035]]], state=connected] 2026-03-24T02:18:08.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:08.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:08.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:08.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:08.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:08.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:08.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:08.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:08.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:08.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:08.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:08.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:08.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:08.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:08.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:08.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:08.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:08.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:08.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:08.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:08.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:08.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=34, tag_tid=8, entry_tid=14038], [object_number=33, tag_tid=8, entry_tid=14037], [object_number=32, tag_tid=8, entry_tid=14036], [object_number=35, tag_tid=8, entry_tid=14035]]], state=connected] 2026-03-24T02:18:08.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:08.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=34, tag_tid=8, entry_tid=14038' 2026-03-24T02:18:08.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=34, tag_tid=8, entry_tid=14038' 2026-03-24T02:18:08.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=34, tag_tid=8, entry_tid=14038' '!=' 'object_number=35, tag_tid=8, entry_tid=13959' 2026-03-24T02:18:08.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:08.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=34, tag_tid=8, entry_tid=14038' = 'object_number=35, tag_tid=8, entry_tid=13959' 2026-03-24T02:18:08.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=34, tag_tid=8, entry_tid=14038' 2026-03-24T02:18:08.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:08.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:08.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:08.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:08.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=34, tag_tid=8, entry_tid=14038' 2026-03-24T02:18:08.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:08.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:08.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:08.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:08.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:08.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:08.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:08.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=34, tag_tid=8, entry_tid=14038' 2026-03-24T02:18:08.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:08.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:08.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=14038 2026-03-24T02:18:08.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 14038 2026-03-24T02:18:08.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:08.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:08.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:08.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:08.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:08.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:08.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:08.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:08.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:08.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:08.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:08.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:08.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:08.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:08.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:08.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:08.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:08.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:08.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:08.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:08.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:08.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:08.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:08.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:08.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:08.320 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:08.320 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:08.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:08.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:08.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:08.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:08.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:08.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:08.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:08.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:08.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:08.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:08.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:08.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:08.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:08.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:08.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:08.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:08.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:08.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:08.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:08.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:08.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=32, tag_tid=8, entry_tid=14120], [object_number=35, tag_tid=8, entry_tid=14119], [object_number=34, tag_tid=8, entry_tid=14118], [object_number=33, tag_tid=8, entry_tid=14117]]], state=connected] 2026-03-24T02:18:08.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:08.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:08.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:08.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:08.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:08.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:08.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:08.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:08.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:08.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:08.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:08.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:08.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:08.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:08.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:08.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:08.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=32, tag_tid=8, entry_tid=14120], [object_number=35, tag_tid=8, entry_tid=14119], [object_number=34, tag_tid=8, entry_tid=14118], [object_number=33, tag_tid=8, entry_tid=14117]]], state=connected] 2026-03-24T02:18:08.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:08.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=32, tag_tid=8, entry_tid=14120' 2026-03-24T02:18:08.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=32, tag_tid=8, entry_tid=14120' 2026-03-24T02:18:08.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=32, tag_tid=8, entry_tid=14120' '!=' 'object_number=34, tag_tid=8, entry_tid=14038' 2026-03-24T02:18:08.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:08.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=32, tag_tid=8, entry_tid=14120' = 'object_number=34, tag_tid=8, entry_tid=14038' 2026-03-24T02:18:08.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=32, tag_tid=8, entry_tid=14120' 2026-03-24T02:18:08.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:08.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:08.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:08.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=32, tag_tid=8, entry_tid=14120' 2026-03-24T02:18:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:08.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:08.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:08.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=32, tag_tid=8, entry_tid=14120' 2026-03-24T02:18:08.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:08.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=14120 2026-03-24T02:18:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 14120 2026-03-24T02:18:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:08.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:08.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:08.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:08.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:08.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:08.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:08.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:08.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:08.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:08.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:08.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:08.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:08.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:08.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:08.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:08.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:08.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:08.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:08.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:08.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:08.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:08.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:08.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:08.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:08.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:08.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:08.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:08.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:08.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:08.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:08.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:08.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:08.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:08.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:08.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:08.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:08.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:08.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:08.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:08.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:08.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:08.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:08.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:08.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:08.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:08.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:08.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:08.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:08.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:08.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:08.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:08.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:08.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:08.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:08.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:08.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:08.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:08.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:08.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:08.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:08.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:08.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:08.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:08.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:08.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:08.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:08.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:08.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:08.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:08.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=35, tag_tid=8, entry_tid=14207], [object_number=34, tag_tid=8, entry_tid=14206], [object_number=33, tag_tid=8, entry_tid=14205], [object_number=32, tag_tid=8, entry_tid=14204]]], state=connected] 2026-03-24T02:18:08.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:08.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:08.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:08.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:08.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:08.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:08.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:08.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:08.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:08.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:08.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:08.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:08.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:08.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:08.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:08.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:08.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:08.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:08.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:08.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:08.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:08.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=35, tag_tid=8, entry_tid=14207], [object_number=34, tag_tid=8, entry_tid=14206], [object_number=33, tag_tid=8, entry_tid=14205], [object_number=32, tag_tid=8, entry_tid=14204]]], state=connected] 2026-03-24T02:18:08.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:08.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=35, tag_tid=8, entry_tid=14207' 2026-03-24T02:18:08.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=35, tag_tid=8, entry_tid=14207' 2026-03-24T02:18:08.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=35, tag_tid=8, entry_tid=14207' '!=' 'object_number=32, tag_tid=8, entry_tid=14120' 2026-03-24T02:18:08.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:08.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=35, tag_tid=8, entry_tid=14207' = 'object_number=32, tag_tid=8, entry_tid=14120' 2026-03-24T02:18:08.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=35, tag_tid=8, entry_tid=14207' 2026-03-24T02:18:08.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:08.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:08.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:08.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:08.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=35, tag_tid=8, entry_tid=14207' 2026-03-24T02:18:08.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:08.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:08.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:08.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:08.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:08.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:08.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:08.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=35, tag_tid=8, entry_tid=14207' 2026-03-24T02:18:08.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:08.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:08.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=14207 2026-03-24T02:18:08.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 14207 2026-03-24T02:18:08.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:08.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:08.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:09.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:09.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:09.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:09.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:09.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:09.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:09.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:09.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:09.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:09.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:09.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:09.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:09.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:09.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:09.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:09.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:09.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:09.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:09.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:09.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:09.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:09.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:09.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:09.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:09.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:09.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:09.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:09.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:09.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:09.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:09.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:09.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:09.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:09.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:09.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:09.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:09.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:09.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:09.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:09.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:09.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:09.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:09.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:09.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:09.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=32, tag_tid=8, entry_tid=14308], [object_number=35, tag_tid=8, entry_tid=14307], [object_number=34, tag_tid=8, entry_tid=14306], [object_number=33, tag_tid=8, entry_tid=14305]]], state=connected] 2026-03-24T02:18:09.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:09.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:09.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:09.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:09.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:09.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:09.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:09.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:09.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:09.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:09.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:09.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:09.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:09.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:09.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:09.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:09.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:09.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:09.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:09.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:09.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:09.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=32, tag_tid=8, entry_tid=14308], [object_number=35, tag_tid=8, entry_tid=14307], [object_number=34, tag_tid=8, entry_tid=14306], [object_number=33, tag_tid=8, entry_tid=14305]]], state=connected] 2026-03-24T02:18:09.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:09.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=32, tag_tid=8, entry_tid=14308' 2026-03-24T02:18:09.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=32, tag_tid=8, entry_tid=14308' 2026-03-24T02:18:09.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=32, tag_tid=8, entry_tid=14308' '!=' 'object_number=35, tag_tid=8, entry_tid=14207' 2026-03-24T02:18:09.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:09.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=32, tag_tid=8, entry_tid=14308' = 'object_number=35, tag_tid=8, entry_tid=14207' 2026-03-24T02:18:09.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=32, tag_tid=8, entry_tid=14308' 2026-03-24T02:18:09.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:09.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:09.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:09.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:09.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=32, tag_tid=8, entry_tid=14308' 2026-03-24T02:18:09.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:09.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:09.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:09.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:09.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:09.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:09.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:09.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=32, tag_tid=8, entry_tid=14308' 2026-03-24T02:18:09.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:09.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:09.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=14308 2026-03-24T02:18:09.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 14308 2026-03-24T02:18:09.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:09.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:09.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:09.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:09.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:09.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:09.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:09.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:09.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:09.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:09.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:09.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:09.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:09.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:09.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:09.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:09.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:09.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:09.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:09.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:09.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:09.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:09.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:09.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:09.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:09.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:09.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:09.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:09.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:09.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:09.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:09.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:09.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:09.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:09.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:09.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:09.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:09.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:09.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:09.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:09.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:09.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:09.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:09.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:09.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:09.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:09.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:09.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:09.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:09.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:09.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:09.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:09.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:09.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:09.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:09.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:09.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:09.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:09.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374], [object_number=33, tag_tid=8, entry_tid=14373], [object_number=32, tag_tid=8, entry_tid=14372]]], state=connected] 2026-03-24T02:18:09.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:09.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:09.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:09.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:09.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:09.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:09.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:09.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:09.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:09.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:09.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:09.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:09.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:09.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:09.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:09.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:09.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:09.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:09.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:09.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:09.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:09.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374], [object_number=33, tag_tid=8, entry_tid=14373], [object_number=32, tag_tid=8, entry_tid=14372]]], state=connected] 2026-03-24T02:18:09.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:09.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=35, tag_tid=8, entry_tid=14375' 2026-03-24T02:18:09.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=35, tag_tid=8, entry_tid=14375' 2026-03-24T02:18:09.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=35, tag_tid=8, entry_tid=14375' '!=' 'object_number=32, tag_tid=8, entry_tid=14308' 2026-03-24T02:18:09.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:09.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=35, tag_tid=8, entry_tid=14375' = 'object_number=32, tag_tid=8, entry_tid=14308' 2026-03-24T02:18:09.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=35, tag_tid=8, entry_tid=14375' 2026-03-24T02:18:09.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:09.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:09.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:09.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=9 2026-03-24T02:18:09.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=35, tag_tid=8, entry_tid=14375' 2026-03-24T02:18:09.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:09.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:09.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=8 2026-03-24T02:18:09.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:09.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:09.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:09.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:09.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=35, tag_tid=8, entry_tid=14375' 2026-03-24T02:18:09.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:09.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:09.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=14375 2026-03-24T02:18:09.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 9 = 8 -a 1 -le 14375 2026-03-24T02:18:09.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:09.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:09.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:09.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:09.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:09.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:09.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:09.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:09.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:09.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:09.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:09.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:09.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:09.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:09.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:09.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:09.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:09.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:09.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:09.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:09.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:09.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:09.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:09.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:09.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:09.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:09.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:09.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:09.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:09.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:09.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:09.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:09.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:09.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:09.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:09.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:09.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:09.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:09.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:09.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:09.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:09.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:09.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:09.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:09.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:09.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:09.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:09.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:09.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:09.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:09.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:09.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:09.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:09.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:09.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:09.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:10.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:10.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:10.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:10.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:10.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:10.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:10.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:10.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:10.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:10.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:10.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:10.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:10.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:10.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:10.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:10.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:10.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:10.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:10.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:10.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:10.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374], [object_number=33, tag_tid=8, entry_tid=14373], [object_number=32, tag_tid=8, entry_tid=14372]]], state=connected] 2026-03-24T02:18:10.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:10.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:10.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:10.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:10.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:10.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:10.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:10.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:10.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:10.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:10.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:10.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:10.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:10.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:10.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:10.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:10.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:10.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:10.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:10.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:10.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:10.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374], [object_number=33, tag_tid=8, entry_tid=14373], [object_number=32, tag_tid=8, entry_tid=14372]]], state=connected] 2026-03-24T02:18:10.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:10.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=35, tag_tid=8, entry_tid=14375' 2026-03-24T02:18:10.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=35, tag_tid=8, entry_tid=14375' 2026-03-24T02:18:10.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=35, tag_tid=8, entry_tid=14375' '!=' 'object_number=35, tag_tid=8, entry_tid=14375' 2026-03-24T02:18:10.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:10.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:18:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:10.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:10.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:10.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:10.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:10.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:10.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:10.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:10.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:10.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:10.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:10.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:10.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:10.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:10.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:10.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:10.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:10.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:10.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:10.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:10.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:10.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:10.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:10.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:10.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:10.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:10.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:10.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:10.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:10.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:10.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:10.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:10.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:10.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:10.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:10.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:10.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:10.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:10.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:10.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:10.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:10.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:10.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:10.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:10.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:10.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:10.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:10.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:10.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:10.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:10.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:10.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:10.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:10.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:10.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 8 2026-03-24T02:18:10.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 8 2026-03-24T02:18:10.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:10.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:10.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=33, tag_tid=9, entry_tid=1], [object_number=32, tag_tid=9, entry_tid=0], [object_number=35, tag_tid=8, entry_tid=14375], [object_number=34, tag_tid=8, entry_tid=14374]]], state=connected] 2026-03-24T02:18:10.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:10.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:10.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=33, tag_tid=9, entry_tid=1' -a 'object_number=33, tag_tid=9, entry_tid=1' = 'object_number=33, tag_tid=9, entry_tid=1' 2026-03-24T02:18:10.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:10.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror test snap3 2026-03-24T02:18:10.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:10.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:10.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:10.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap3 2026-03-24T02:18:10.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:18:10.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:18:10.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:18:11.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/test@snap3 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'test': 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 5 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c9689d5e36 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c9689d5e36 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:15:23 2026 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:17:03 2026 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:17:27 2026 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c9689d5e36 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:18:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:18:11.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:18:11.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:11.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n snap2-clean ']' 2026-03-24T02:18:11.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap2-clean 2026-03-24T02:18:11.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:11.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:11.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap2-clean 2026-03-24T02:18:11.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:18:11.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:18:11.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:18:11.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:18:11.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:18:12.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:18:12.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:18:13.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:18:13.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:18:13.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:18:14.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:14.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap3 2026-03-24T02:18:14.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:14.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:14.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap3 2026-03-24T02:18:14.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:18:14.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:18:14.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:18:14.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:18:14.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:18:15.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:18:15.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:18:15.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:18:15.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:18:15.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:18:16.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:16.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ clean_snap_name=snap3-clean 2026-03-24T02:18:16.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap3-clean 2026-03-24T02:18:16.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:18:16.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:16.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:16.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap3-clean 2026-03-24T02:18:16.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3-clean --debug-rbd=20 --debug-journaler=20 2026-03-24T02:18:17.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:18:17.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror test 2026-03-24T02:18:17.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:18:17.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:17.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:17.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:18:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=5 2026-03-24T02:18:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:18:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-03-24T02:18:18.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 28399616 bytes 2026-03-24T02:18:19.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 68199424 bytes 2026-03-24T02:18:20.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 68810752 bytes 2026-03-24T02:18:21.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:400 IOs, 85923840 bytes 2026-03-24T02:18:22.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:18:22.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:18:22.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:18:22.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:22.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-24T02:18:22.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:22.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:22.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:22.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:18:22.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:18:22.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:18:22.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:22.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:18:23.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-24T02:18:23.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:23.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:23.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:23.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:18:23.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:18:23.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:18:23.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:18:23.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:18:23.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-24T02:18:23.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='test: 2026-03-24T02:18:23.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: global_id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:18:23.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+replaying 2026-03-24T02:18:23.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: replaying, {"bytes_per_second":2696525.7,"entries_behind_primary":0,"entries_per_second":168.8,"non_primary_position":{"entry_tid":1,"object_number":33,"tag_tid":9},"primary_position":{"entry_tid":1,"object_number":33,"tag_tid":9}} 2026-03-24T02:18:23.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: service: mirror.0 on vm07 2026-03-24T02:18:23.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:18:21 2026-03-24T02:18:23.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: peer_sites: 2026-03-24T02:18:23.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: name: cluster2 2026-03-24T02:18:23.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+stopped 2026-03-24T02:18:23.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: local image is primary 2026-03-24T02:18:23.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:18:04' 2026-03-24T02:18:23.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ state: .*up+replaying' 2026-03-24T02:18:23.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: state: up+replaying 2026-03-24T02:18:23.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ description: .*' 2026-03-24T02:18:23.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: description: replaying, {"bytes_per_second":2696525.7,"entries_behind_primary":0,"entries_per_second":168.8,"non_primary_position":{"entry_tid":1,"object_number":33,"tag_tid":9},"primary_position":{"entry_tid":1,"object_number":33,"tag_tid":9}} 2026-03-24T02:18:23.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:18:23.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:18:23.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^up+' 2026-03-24T02:18:23.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:18:23.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep 'service: *mirror..* on ' 2026-03-24T02:18:23.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: service: mirror.0 on vm07 2026-03-24T02:18:23.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update 2026-03-24T02:18:23.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-24T02:18:23.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:18:21' 2026-03-24T02:18:23.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-24 02:18:21' 2026-03-24T02:18:23.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:23.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:23.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:23.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:18:23.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'prev_last_update=2026-03-24 02:18:21' 2026-03-24T02:18:23.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:18:23.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:18:23.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-24T02:18:23.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='OKOKOK142254303mirror.2vm0720.2.0-712-g70f8415bfalseOK42264305mirror.3vm0720.2.0-712-g70f8415bfalseOK42944297mirror.0vm0720.2.0-712-g70f8415btrueOK43554358mirror.1vm0720.2.0-712-g70f8415bfalseOKtest2d4a20c9-f6e9-4ce8-9d64-84febc265629up+replayingreplaying, {"bytes_per_second":2696525.7,"entries_behind_primary":0,"entries_per_second":168.8,"non_primary_position":{"entry_tid":1,"object_number":33,"tag_tid":9},"primary_position":{"entry_tid":1,"object_number":33,"tag_tid":9}}42944297mirror.0vm072026-03-24 02:18:21cluster26c644ee3-7db1-4d97-9283-48b1f8a82096up+stoppedlocal image is primary2026-03-24 02:18:04' 2026-03-24T02:18:23.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update state 2026-03-24T02:18:23.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-24T02:18:23.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:18:21' 2026-03-24T02:18:23.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-24T02:18:23.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=up+replaying 2026-03-24T02:18:23.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:18:23.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep up+replaying 2026-03-24T02:18:23.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:18:23.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:23.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:23.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap4 2026-03-24T02:18:23.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap4 2026-03-24T02:18:23.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:18:23.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:23.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:23.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap4 2026-03-24T02:18:23.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:18:24.927 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.467933289876464) > weight (0.25) 2026-03-24T02:18:24.927 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.3148498429237033) > weight (0.25) 2026-03-24T02:18:24.927 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.662199290523397) > weight (0.25) 2026-03-24T02:18:24.927 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.5026058271136461) > weight (0.25) 2026-03-24T02:18:24.927 INFO:tasks.rbd_mirror_thrash:waiting for 117.01872801068922 secs before thrashing 2026-03-24T02:18:51.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-24T02:18:51.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:51.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:51.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:51.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-24T02:18:51.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:51.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:51.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:51.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:18:51.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:18:51.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:51.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:18:52.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-24T02:18:52.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:52.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:52.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:52.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:18:52.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:18:52.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:18:52.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-24T02:18:52.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:18:52.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:18:52.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:52.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:18:52.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:18:52.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:18:52.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:52.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:18:52.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:52.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:18:52.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:18:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:52.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:52.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:52.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:52.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-24T02:18:52.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:18:52.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:18:52.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:18:52.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:18:52.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:52.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:52.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:52.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:52.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:52.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:52.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:52.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:52.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:52.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:52.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:52.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:52.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:52.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:52.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:52.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:52.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:52.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:52.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:52.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:52.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:52.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:52.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:52.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:52.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:52.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:52.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:52.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:52.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:52.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:52.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:52.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:52.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:52.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:52.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:52.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:52.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:52.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:52.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:52.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:52.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:52.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:52.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:52.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:52.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:52.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:52.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:52.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:52.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:52.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:52.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:52.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:52.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:52.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 9 2026-03-24T02:18:52.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 9 2026-03-24T02:18:52.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:52.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=12, entry_tid=1], [object_number=36, tag_tid=12, entry_tid=0], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158]]], state=connected] 2026-03-24T02:18:52.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=36, tag_tid=11, entry_tid=6160], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158], [object_number=37, tag_tid=11, entry_tid=6157]]], state=connected] 2026-03-24T02:18:52.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:52.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=37, tag_tid=12, entry_tid=1' 2026-03-24T02:18:52.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:52.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:52.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:52.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:52.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:52.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:52.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:52.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:52.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:52.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:52.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:52.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:52.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:52.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:52.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:52.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 9 2026-03-24T02:18:52.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 9 2026-03-24T02:18:52.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:52.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=12, entry_tid=1], [object_number=36, tag_tid=12, entry_tid=0], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158]]], state=connected] 2026-03-24T02:18:52.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=36, tag_tid=11, entry_tid=6160], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158], [object_number=37, tag_tid=11, entry_tid=6157]]], state=connected] 2026-03-24T02:18:52.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=36, tag_tid=11, entry_tid=6160' 2026-03-24T02:18:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=37, tag_tid=12, entry_tid=1' -a 'object_number=37, tag_tid=12, entry_tid=1' = 'object_number=36, tag_tid=11, entry_tid=6160' 2026-03-24T02:18:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=36, tag_tid=11, entry_tid=6160' '!=' '' 2026-03-24T02:18:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:18:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=36, tag_tid=11, entry_tid=6160' = '' 2026-03-24T02:18:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=36, tag_tid=11, entry_tid=6160' 2026-03-24T02:18:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=37, tag_tid=12, entry_tid=1' 2026-03-24T02:18:52.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:52.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:52.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=12 2026-03-24T02:18:52.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=36, tag_tid=11, entry_tid=6160' 2026-03-24T02:18:52.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:18:52.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=11 2026-03-24T02:18:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=37, tag_tid=12, entry_tid=1' 2026-03-24T02:18:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:52.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:18:52.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=36, tag_tid=11, entry_tid=6160' 2026-03-24T02:18:52.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:18:52.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:18:52.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=6160 2026-03-24T02:18:52.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 12 = 11 -a 1 -le 6160 2026-03-24T02:18:52.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:18:52.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:52.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:18:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:53.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:53.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:53.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:53.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:53.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:53.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:53.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:53.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:53.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:53.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:53.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:53.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:53.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:53.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:53.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:53.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:53.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:53.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:53.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:53.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:53.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:53.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:53.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:53.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:53.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:53.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:53.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:53.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:53.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:53.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:53.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:53.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:53.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:53.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:53.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 9 2026-03-24T02:18:53.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 9 2026-03-24T02:18:53.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:53.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=12, entry_tid=1], [object_number=36, tag_tid=12, entry_tid=0], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158]]], state=connected] 2026-03-24T02:18:53.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=36, tag_tid=11, entry_tid=6160], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158], [object_number=37, tag_tid=11, entry_tid=6157]]], state=connected] 2026-03-24T02:18:53.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:53.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=37, tag_tid=12, entry_tid=1' 2026-03-24T02:18:53.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:53.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:53.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:53.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:53.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:53.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:53.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:53.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:53.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:53.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:53.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:53.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:53.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:53.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:53.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:53.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 9 2026-03-24T02:18:53.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 9 2026-03-24T02:18:53.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:53.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=12, entry_tid=1], [object_number=36, tag_tid=12, entry_tid=0], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158]]], state=connected] 2026-03-24T02:18:53.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=36, tag_tid=11, entry_tid=6160], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158], [object_number=37, tag_tid=11, entry_tid=6157]]], state=connected] 2026-03-24T02:18:53.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:53.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=36, tag_tid=11, entry_tid=6160' 2026-03-24T02:18:53.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=37, tag_tid=12, entry_tid=1' -a 'object_number=37, tag_tid=12, entry_tid=1' = 'object_number=36, tag_tid=11, entry_tid=6160' 2026-03-24T02:18:53.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=36, tag_tid=11, entry_tid=6160' '!=' 'object_number=36, tag_tid=11, entry_tid=6160' 2026-03-24T02:18:53.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:53.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:18:53.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:53.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:53.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:53.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:53.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:53.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:53.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:53.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:53.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:53.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:53.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:53.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:53.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:53.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:53.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:53.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:53.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:53.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:53.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:53.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:53.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:53.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:53.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:53.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:53.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:53.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:53.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:53.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:53.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:53.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:53.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 9 2026-03-24T02:18:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 9 2026-03-24T02:18:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=12, entry_tid=1], [object_number=36, tag_tid=12, entry_tid=0], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158]]], state=connected] 2026-03-24T02:18:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=36, tag_tid=11, entry_tid=6160], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158], [object_number=37, tag_tid=11, entry_tid=6157]]], state=connected] 2026-03-24T02:18:53.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:53.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=37, tag_tid=12, entry_tid=1' 2026-03-24T02:18:53.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:53.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:53.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:53.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:53.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:53.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:53.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:53.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:53.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:53.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:53.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:53.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:53.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:53.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:53.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 9 2026-03-24T02:18:53.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 9 2026-03-24T02:18:53.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:53.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=12, entry_tid=1], [object_number=36, tag_tid=12, entry_tid=0], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158]]], state=connected] 2026-03-24T02:18:53.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=36, tag_tid=11, entry_tid=6160], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158], [object_number=37, tag_tid=11, entry_tid=6157]]], state=connected] 2026-03-24T02:18:53.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:53.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=36, tag_tid=11, entry_tid=6160' 2026-03-24T02:18:53.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=37, tag_tid=12, entry_tid=1' -a 'object_number=37, tag_tid=12, entry_tid=1' = 'object_number=36, tag_tid=11, entry_tid=6160' 2026-03-24T02:18:53.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=36, tag_tid=11, entry_tid=6160' '!=' 'object_number=36, tag_tid=11, entry_tid=6160' 2026-03-24T02:18:53.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:53.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:18:54.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:54.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:54.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:54.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:54.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:54.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:54.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:54.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:54.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:54.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:54.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:54.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:54.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:54.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:54.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:54.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:54.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:54.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:54.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:54.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:54.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:54.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:54.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:54.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:54.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:54.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:54.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:54.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:54.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:54.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:54.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:54.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:54.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:54.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:54.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:54.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:54.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:54.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:54.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:54.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:54.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:54.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:54.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:54.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:54.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:54.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:54.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:54.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:54.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 9 2026-03-24T02:18:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 9 2026-03-24T02:18:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=12, entry_tid=1], [object_number=36, tag_tid=12, entry_tid=0], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158]]], state=connected] 2026-03-24T02:18:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=36, tag_tid=11, entry_tid=6160], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158], [object_number=37, tag_tid=11, entry_tid=6157]]], state=connected] 2026-03-24T02:18:54.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:54.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=37, tag_tid=12, entry_tid=1' 2026-03-24T02:18:54.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:54.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:54.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:54.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:54.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:54.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:54.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:54.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:54.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:54.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:54.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:54.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:54.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:54.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:54.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:54.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 9 2026-03-24T02:18:54.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 9 2026-03-24T02:18:54.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:54.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=12, entry_tid=1], [object_number=36, tag_tid=12, entry_tid=0], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158]]], state=connected] 2026-03-24T02:18:54.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=36, tag_tid=11, entry_tid=6160], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158], [object_number=37, tag_tid=11, entry_tid=6157]]], state=connected] 2026-03-24T02:18:54.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:54.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=36, tag_tid=11, entry_tid=6160' 2026-03-24T02:18:54.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=37, tag_tid=12, entry_tid=1' -a 'object_number=37, tag_tid=12, entry_tid=1' = 'object_number=36, tag_tid=11, entry_tid=6160' 2026-03-24T02:18:54.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=36, tag_tid=11, entry_tid=6160' '!=' 'object_number=36, tag_tid=11, entry_tid=6160' 2026-03-24T02:18:54.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:54.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:18:56.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:56.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:56.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:56.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:56.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:56.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:56.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:56.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:56.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:56.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:56.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:56.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:56.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:56.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:56.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:56.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:56.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:56.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:56.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:56.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:56.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:56.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:56.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:56.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:56.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:56.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:56.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:56.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:56.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:56.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:56.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:56.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:56.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:56.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:56.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:56.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:56.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:56.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:56.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:56.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:56.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:56.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:56.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:56.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 9 2026-03-24T02:18:56.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 9 2026-03-24T02:18:56.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:56.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=12, entry_tid=1], [object_number=36, tag_tid=12, entry_tid=0], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158]]], state=connected] 2026-03-24T02:18:56.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=36, tag_tid=11, entry_tid=6160], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158], [object_number=37, tag_tid=11, entry_tid=6157]]], state=connected] 2026-03-24T02:18:56.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:56.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=37, tag_tid=12, entry_tid=1' 2026-03-24T02:18:56.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:56.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:56.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:56.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:56.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:56.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:56.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:56.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:56.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:56.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:56.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:56.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:56.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:56.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:56.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:56.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 9 2026-03-24T02:18:56.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 9 2026-03-24T02:18:56.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:56.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=12, entry_tid=1], [object_number=36, tag_tid=12, entry_tid=0], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158]]], state=connected] 2026-03-24T02:18:56.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=36, tag_tid=11, entry_tid=6160], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158], [object_number=37, tag_tid=11, entry_tid=6157]]], state=connected] 2026-03-24T02:18:56.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:56.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=36, tag_tid=11, entry_tid=6160' 2026-03-24T02:18:56.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=37, tag_tid=12, entry_tid=1' -a 'object_number=37, tag_tid=12, entry_tid=1' = 'object_number=36, tag_tid=11, entry_tid=6160' 2026-03-24T02:18:56.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=36, tag_tid=11, entry_tid=6160' '!=' 'object_number=36, tag_tid=11, entry_tid=6160' 2026-03-24T02:18:56.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:18:56.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:18:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:18:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:18:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:18:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:18:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:18:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:18:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:18:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:18:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:18:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:18:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:18:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:18:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:18:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:18:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:18:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:18:58.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:18:58.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:58.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:58.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:18:58.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:18:58.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:58.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:18:58.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:58.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:58.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:58.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:18:58.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:58.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:58.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:58.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:18:58.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:58.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:58.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:58.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:58.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:58.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:58.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 9 2026-03-24T02:18:58.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 9 2026-03-24T02:18:58.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:58.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=12, entry_tid=1], [object_number=36, tag_tid=12, entry_tid=0], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158]]], state=connected] 2026-03-24T02:18:58.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=37, tag_tid=12, entry_tid=1], [object_number=36, tag_tid=12, entry_tid=0], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158]]], state=connected] 2026-03-24T02:18:58.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=37, tag_tid=12, entry_tid=1' 2026-03-24T02:18:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:18:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:18:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:18:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:18:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:18:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:18:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:18:58.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:18:58.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:18:58.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:58.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:18:58.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:58.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 9 2026-03-24T02:18:58.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 9 2026-03-24T02:18:58.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:18:58.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=12, entry_tid=1], [object_number=36, tag_tid=12, entry_tid=0], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158]]], state=connected] 2026-03-24T02:18:58.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=37, tag_tid=12, entry_tid=1], [object_number=36, tag_tid=12, entry_tid=0], [object_number=39, tag_tid=11, entry_tid=6159], [object_number=38, tag_tid=11, entry_tid=6158]]], state=connected] 2026-03-24T02:18:58.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:18:58.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=37, tag_tid=12, entry_tid=1' 2026-03-24T02:18:58.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=37, tag_tid=12, entry_tid=1' -a 'object_number=37, tag_tid=12, entry_tid=1' = 'object_number=37, tag_tid=12, entry_tid=1' 2026-03-24T02:18:58.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:58.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror test snap4 2026-03-24T02:18:58.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:18:58.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:58.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:58.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap4 2026-03-24T02:18:58.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:18:58.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:18:58.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:18:59.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/test@snap4 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'test': 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 7 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c9689d5e36 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c9689d5e36 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:15:23 2026 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:18:12 2026 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:18:28 2026 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c9689d5e36 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:18:59.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:18:59.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:18:59.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n snap3-clean ']' 2026-03-24T02:18:59.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap3-clean 2026-03-24T02:18:59.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:18:59.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:18:59.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap3-clean 2026-03-24T02:18:59.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:18:59.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:18:59.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:18:59.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:18:59.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:19:00.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:19:00.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:19:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:19:01.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:19:01.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:19:01.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:01.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap4 2026-03-24T02:19:01.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:01.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:01.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap4 2026-03-24T02:19:01.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:19:01.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:19:01.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:19:01.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:19:01.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:19:01.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:19:01.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:19:02.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:19:02.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:19:03.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:19:03.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:03.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ clean_snap_name=snap4-clean 2026-03-24T02:19:03.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap4-clean 2026-03-24T02:19:03.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:19:03.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:03.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:03.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap4-clean 2026-03-24T02:19:03.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4-clean --debug-rbd=20 --debug-journaler=20 2026-03-24T02:19:03.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:19:03.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror test 2026-03-24T02:19:03.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:19:03.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:03.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:03.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:19:03.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=14 2026-03-24T02:19:03.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:19:03.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-03-24T02:19:05.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 31590400 bytes 2026-03-24T02:19:08.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 116798464 bytes 2026-03-24T02:19:13.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 246679552 bytes 2026-03-24T02:19:17.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:400 IOs, 288074752 bytes 2026-03-24T02:19:17.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:19:17.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:19:17.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:19:17.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:17.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-24T02:19:17.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:17.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:17.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:17.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:19:17.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:19:17.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:19:17.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:17.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:19:18.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-24T02:19:18.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:18.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:18.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:18.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:19:18.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:19:18.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:19:18.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:19:18.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:19:18.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-24T02:19:19.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='test: 2026-03-24T02:19:19.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: global_id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:19:19.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+replaying 2026-03-24T02:19:19.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: replaying, {"bytes_per_second":3207556.83,"entries_behind_primary":0,"entries_per_second":205.43,"non_primary_position":{"entry_tid":6159,"object_number":39,"tag_tid":11},"primary_position":{"entry_tid":1,"object_number":33,"tag_tid":10}} 2026-03-24T02:19:19.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: service: mirror.0 on vm07 2026-03-24T02:19:19.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:18:51 2026-03-24T02:19:19.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: peer_sites: 2026-03-24T02:19:19.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: name: cluster2 2026-03-24T02:19:19.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+stopped 2026-03-24T02:19:19.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: local image is primary 2026-03-24T02:19:19.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:19:04' 2026-03-24T02:19:19.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ state: .*up+replaying' 2026-03-24T02:19:19.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: state: up+replaying 2026-03-24T02:19:19.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ description: .*' 2026-03-24T02:19:19.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: description: replaying, {"bytes_per_second":3207556.83,"entries_behind_primary":0,"entries_per_second":205.43,"non_primary_position":{"entry_tid":6159,"object_number":39,"tag_tid":11},"primary_position":{"entry_tid":1,"object_number":33,"tag_tid":10}} 2026-03-24T02:19:19.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:19:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:19:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^up+' 2026-03-24T02:19:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:19:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep 'service: *mirror..* on ' 2026-03-24T02:19:19.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: service: mirror.0 on vm07 2026-03-24T02:19:19.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update 2026-03-24T02:19:19.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-24T02:19:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:18:51' 2026-03-24T02:19:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-24 02:18:51' 2026-03-24T02:19:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:19:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'prev_last_update=2026-03-24 02:18:51' 2026-03-24T02:19:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:19:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:19:19.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-24T02:19:19.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='OKOKOK142254303mirror.2vm0720.2.0-712-g70f8415bfalseOK42264305mirror.3vm0720.2.0-712-g70f8415bfalseOK42944297mirror.0vm0720.2.0-712-g70f8415btrueOK43554358mirror.1vm0720.2.0-712-g70f8415bfalseOKtest2d4a20c9-f6e9-4ce8-9d64-84febc265629up+replayingreplaying, {"bytes_per_second":3207556.83,"entries_behind_primary":0,"entries_per_second":205.43,"non_primary_position":{"entry_tid":6159,"object_number":39,"tag_tid":11},"primary_position":{"entry_tid":1,"object_number":33,"tag_tid":10}}42944297mirror.0vm072026-03-24 02:18:51cluster26c644ee3-7db1-4d97-9283-48b1f8a82096up+stoppedlocal image is primary2026-03-24 02:19:04' 2026-03-24T02:19:19.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update state 2026-03-24T02:19:19.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-24T02:19:19.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:18:51' 2026-03-24T02:19:19.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-24T02:19:19.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=up+replaying 2026-03-24T02:19:19.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:19:19.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep up+replaying 2026-03-24T02:19:19.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:19:19.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:19.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:19.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap5 2026-03-24T02:19:19.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap5 2026-03-24T02:19:19.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:19:19.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:19.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:19.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap5 2026-03-24T02:19:19.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:19:23.336 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7823390601706772) > weight (0.25) 2026-03-24T02:19:23.336 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9351140792638145) > weight (0.25) 2026-03-24T02:19:23.336 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.6788452234424833) > weight (0.25) 2026-03-24T02:19:23.336 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.7607050205136471) > weight (0.25) 2026-03-24T02:19:23.336 INFO:tasks.rbd_mirror_thrash:waiting for 90.32705182481462 secs before thrashing 2026-03-24T02:19:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-24T02:19:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-24T02:19:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:19:46.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:19:46.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:46.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:19:47.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-24T02:19:47.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:47.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:47.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:47.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:19:47.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:19:47.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:19:47.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-24T02:19:47.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:19:47.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:19:47.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:19:47.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:47.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:19:47.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:19:47.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:47.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:19:47.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:47.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:19:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-24T02:19:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:19:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:19:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:19:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:19:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:19:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:19:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:19:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:19:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:19:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:19:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:19:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:19:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:47.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:47.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:19:47.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:19:47.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:19:47.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:19:47.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:47.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:47.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:47.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:47.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-24T02:19:47.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:19:47.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:19:47.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:19:47.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:47.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:19:47.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:19:47.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:47.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:47.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:48.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:48.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:48.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:48.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:48.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:48.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:48.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:48.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:48.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:48.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:48.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:48.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:48.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:48.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:48.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:48.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:48.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:48.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:48.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:48.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:48.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:48.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:48.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:48.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:48.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:48.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:48.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:48.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:48.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:48.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:48.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:48.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:48.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:48.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:48.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:48.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:48.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:48.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:48.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:48.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:48.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:48.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:48.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:48.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:48.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:48.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:48.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:48.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:48.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:48.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:48.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:48.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:48.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:48.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:48.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:48.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:48.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:48.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:48.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:48.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:48.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:48.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:48.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:48.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:48.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:48.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:48.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:48.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:48.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:48.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:48.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:48.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:48.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:48.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=53, tag_tid=14, entry_tid=14969], [object_number=52, tag_tid=14, entry_tid=14968], [object_number=55, tag_tid=14, entry_tid=14967], [object_number=54, tag_tid=14, entry_tid=14966]]], state=connected] 2026-03-24T02:19:48.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:48.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:48.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:48.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:48.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:48.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:48.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:48.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:48.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:48.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:48.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:48.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:48.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:48.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:48.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:48.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:48.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:48.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:48.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:48.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:48.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:48.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=53, tag_tid=14, entry_tid=14969], [object_number=52, tag_tid=14, entry_tid=14968], [object_number=55, tag_tid=14, entry_tid=14967], [object_number=54, tag_tid=14, entry_tid=14966]]], state=connected] 2026-03-24T02:19:48.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:48.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=53, tag_tid=14, entry_tid=14969' 2026-03-24T02:19:48.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=53, tag_tid=14, entry_tid=14969' 2026-03-24T02:19:48.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=53, tag_tid=14, entry_tid=14969' '!=' '' 2026-03-24T02:19:48.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:48.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=53, tag_tid=14, entry_tid=14969' = '' 2026-03-24T02:19:48.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=53, tag_tid=14, entry_tid=14969' 2026-03-24T02:19:48.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:48.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:48.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:48.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:48.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=53, tag_tid=14, entry_tid=14969' 2026-03-24T02:19:48.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:48.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:48.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:48.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:48.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:48.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:48.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:48.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=53, tag_tid=14, entry_tid=14969' 2026-03-24T02:19:48.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:48.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:48.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=14969 2026-03-24T02:19:48.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 14969 2026-03-24T02:19:48.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:48.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:48.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:48.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:48.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:48.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:48.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:48.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:48.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:48.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:48.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:48.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:48.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:48.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:48.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:48.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:48.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:48.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:48.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:48.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:48.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:48.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:48.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:48.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:48.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:48.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:48.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:48.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:48.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:48.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:48.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:48.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:48.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:48.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:48.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:48.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:48.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:48.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:48.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:48.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:48.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:48.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:48.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:48.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:48.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:48.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:48.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:48.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:48.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:48.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:48.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:48.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:48.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:48.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:48.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:48.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:48.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:48.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:48.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:48.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:48.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:48.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:48.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:48.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:48.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:48.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=55, tag_tid=14, entry_tid=15087], [object_number=54, tag_tid=14, entry_tid=15086], [object_number=53, tag_tid=14, entry_tid=15085], [object_number=52, tag_tid=14, entry_tid=15084]]], state=connected] 2026-03-24T02:19:48.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:48.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:48.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:48.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:48.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:48.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:48.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:48.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:48.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:48.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:48.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:48.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:48.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:48.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:48.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:48.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:48.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:48.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:48.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:48.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:48.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:48.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=55, tag_tid=14, entry_tid=15087], [object_number=54, tag_tid=14, entry_tid=15086], [object_number=53, tag_tid=14, entry_tid=15085], [object_number=52, tag_tid=14, entry_tid=15084]]], state=connected] 2026-03-24T02:19:48.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:48.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=55, tag_tid=14, entry_tid=15087' 2026-03-24T02:19:48.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=55, tag_tid=14, entry_tid=15087' 2026-03-24T02:19:48.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=55, tag_tid=14, entry_tid=15087' '!=' 'object_number=53, tag_tid=14, entry_tid=14969' 2026-03-24T02:19:48.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:48.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=55, tag_tid=14, entry_tid=15087' = 'object_number=53, tag_tid=14, entry_tid=14969' 2026-03-24T02:19:48.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=55, tag_tid=14, entry_tid=15087' 2026-03-24T02:19:48.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:48.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:48.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:48.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:48.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=55, tag_tid=14, entry_tid=15087' 2026-03-24T02:19:48.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:48.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:48.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:48.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:48.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:48.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:48.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:48.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=55, tag_tid=14, entry_tid=15087' 2026-03-24T02:19:48.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:48.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:48.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=15087 2026-03-24T02:19:48.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 15087 2026-03-24T02:19:48.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:48.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:48.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:49.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:49.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:49.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:49.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:49.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:49.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:49.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:49.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:49.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:49.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:49.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:49.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:49.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:49.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:49.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:49.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:49.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:49.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:49.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:49.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:49.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:49.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:49.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:49.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:49.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:49.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:49.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:49.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:49.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:49.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:49.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:49.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:49.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:49.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:49.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:49.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:49.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:49.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:49.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=52, tag_tid=14, entry_tid=15220], [object_number=55, tag_tid=14, entry_tid=15219], [object_number=54, tag_tid=14, entry_tid=15218], [object_number=53, tag_tid=14, entry_tid=15217]]], state=connected] 2026-03-24T02:19:49.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:49.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:49.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:49.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:49.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:49.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:49.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:49.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:49.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:49.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:49.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:49.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:49.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:49.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:49.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:49.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:49.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:49.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:49.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:49.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:49.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:49.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=52, tag_tid=14, entry_tid=15220], [object_number=55, tag_tid=14, entry_tid=15219], [object_number=54, tag_tid=14, entry_tid=15218], [object_number=53, tag_tid=14, entry_tid=15217]]], state=connected] 2026-03-24T02:19:49.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:49.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=52, tag_tid=14, entry_tid=15220' 2026-03-24T02:19:49.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=52, tag_tid=14, entry_tid=15220' 2026-03-24T02:19:49.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=52, tag_tid=14, entry_tid=15220' '!=' 'object_number=55, tag_tid=14, entry_tid=15087' 2026-03-24T02:19:49.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:49.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=52, tag_tid=14, entry_tid=15220' = 'object_number=55, tag_tid=14, entry_tid=15087' 2026-03-24T02:19:49.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=52, tag_tid=14, entry_tid=15220' 2026-03-24T02:19:49.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:49.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:49.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:49.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:49.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=52, tag_tid=14, entry_tid=15220' 2026-03-24T02:19:49.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:49.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:49.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:49.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:49.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:49.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:49.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:49.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=52, tag_tid=14, entry_tid=15220' 2026-03-24T02:19:49.286 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:49.286 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:49.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=15220 2026-03-24T02:19:49.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 15220 2026-03-24T02:19:49.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:49.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:49.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:49.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:49.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:49.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:49.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:49.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:49.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:49.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:49.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:49.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:49.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:49.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:49.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:49.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:49.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:49.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:49.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:49.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:49.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:49.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:49.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:49.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:49.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:49.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:49.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:49.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:49.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:49.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:49.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:49.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:49.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:49.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:49.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:49.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:49.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:49.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:49.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:49.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:49.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:49.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:49.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:49.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:49.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:49.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:49.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=55, tag_tid=14, entry_tid=15371], [object_number=54, tag_tid=14, entry_tid=15370], [object_number=53, tag_tid=14, entry_tid=15369], [object_number=52, tag_tid=14, entry_tid=15368]]], state=connected] 2026-03-24T02:19:49.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:49.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:49.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:49.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:49.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:49.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:49.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:49.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:49.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:49.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:49.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:49.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:49.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:49.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:49.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:49.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:49.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:49.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:49.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:49.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:49.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:49.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=55, tag_tid=14, entry_tid=15371], [object_number=54, tag_tid=14, entry_tid=15370], [object_number=53, tag_tid=14, entry_tid=15369], [object_number=52, tag_tid=14, entry_tid=15368]]], state=connected] 2026-03-24T02:19:49.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:49.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=55, tag_tid=14, entry_tid=15371' 2026-03-24T02:19:49.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=55, tag_tid=14, entry_tid=15371' 2026-03-24T02:19:49.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=55, tag_tid=14, entry_tid=15371' '!=' 'object_number=52, tag_tid=14, entry_tid=15220' 2026-03-24T02:19:49.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:49.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=55, tag_tid=14, entry_tid=15371' = 'object_number=52, tag_tid=14, entry_tid=15220' 2026-03-24T02:19:49.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=55, tag_tid=14, entry_tid=15371' 2026-03-24T02:19:49.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:49.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:49.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:49.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:49.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=55, tag_tid=14, entry_tid=15371' 2026-03-24T02:19:49.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:49.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:49.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:49.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:49.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:49.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:49.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:49.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=55, tag_tid=14, entry_tid=15371' 2026-03-24T02:19:49.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:49.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:49.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=15371 2026-03-24T02:19:49.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 15371 2026-03-24T02:19:49.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:49.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:49.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:49.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:49.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:49.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:49.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:49.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:49.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:49.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:49.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:49.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:49.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:49.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:49.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:49.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:49.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:49.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:49.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:49.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:49.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:49.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:49.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:49.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:49.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:49.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:49.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:49.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:49.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:49.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:49.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:49.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:49.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:49.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:49.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:49.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:49.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:50.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:50.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:50.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:50.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:50.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:50.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:50.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:50.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:50.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:50.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:50.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:50.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:50.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:50.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:50.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:50.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:50.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:50.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:50.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:50.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:50.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=55, tag_tid=14, entry_tid=15507], [object_number=54, tag_tid=14, entry_tid=15506], [object_number=53, tag_tid=14, entry_tid=15505], [object_number=52, tag_tid=14, entry_tid=15504]]], state=connected] 2026-03-24T02:19:50.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:50.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:50.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:50.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:50.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:50.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:50.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:50.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:50.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:50.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:50.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:50.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:50.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:50.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:50.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:50.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:50.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:50.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:50.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=55, tag_tid=14, entry_tid=15507], [object_number=54, tag_tid=14, entry_tid=15506], [object_number=53, tag_tid=14, entry_tid=15505], [object_number=52, tag_tid=14, entry_tid=15504]]], state=connected] 2026-03-24T02:19:50.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:50.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=55, tag_tid=14, entry_tid=15507' 2026-03-24T02:19:50.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=55, tag_tid=14, entry_tid=15507' 2026-03-24T02:19:50.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=55, tag_tid=14, entry_tid=15507' '!=' 'object_number=55, tag_tid=14, entry_tid=15371' 2026-03-24T02:19:50.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:50.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=55, tag_tid=14, entry_tid=15507' = 'object_number=55, tag_tid=14, entry_tid=15371' 2026-03-24T02:19:50.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=55, tag_tid=14, entry_tid=15507' 2026-03-24T02:19:50.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:50.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:50.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:50.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:50.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=55, tag_tid=14, entry_tid=15507' 2026-03-24T02:19:50.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:50.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:50.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:50.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:50.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:50.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:50.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:50.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=55, tag_tid=14, entry_tid=15507' 2026-03-24T02:19:50.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:50.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:50.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=15507 2026-03-24T02:19:50.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 15507 2026-03-24T02:19:50.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:50.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:50.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:50.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:50.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:50.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:50.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:50.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:50.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:50.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:50.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:50.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:50.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:50.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:50.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:50.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:50.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:50.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:50.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:50.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:50.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:50.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:50.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:50.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:50.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:50.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:50.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:50.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:50.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:50.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:50.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:50.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:50.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:50.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:50.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:50.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:50.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:50.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:50.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:50.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=52, tag_tid=14, entry_tid=15700], [object_number=55, tag_tid=14, entry_tid=15699], [object_number=54, tag_tid=14, entry_tid=15698], [object_number=53, tag_tid=14, entry_tid=15697]]], state=connected] 2026-03-24T02:19:50.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:50.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:50.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:50.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:50.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:50.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:50.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:50.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:50.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=52, tag_tid=14, entry_tid=15700], [object_number=55, tag_tid=14, entry_tid=15699], [object_number=54, tag_tid=14, entry_tid=15698], [object_number=53, tag_tid=14, entry_tid=15697]]], state=connected] 2026-03-24T02:19:50.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:50.745 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=52, tag_tid=14, entry_tid=15700' 2026-03-24T02:19:50.745 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=52, tag_tid=14, entry_tid=15700' 2026-03-24T02:19:50.745 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=52, tag_tid=14, entry_tid=15700' '!=' 'object_number=55, tag_tid=14, entry_tid=15507' 2026-03-24T02:19:50.745 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:50.745 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=52, tag_tid=14, entry_tid=15700' = 'object_number=55, tag_tid=14, entry_tid=15507' 2026-03-24T02:19:50.745 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=52, tag_tid=14, entry_tid=15700' 2026-03-24T02:19:50.745 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:50.745 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:50.745 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:50.746 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:50.747 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=52, tag_tid=14, entry_tid=15700' 2026-03-24T02:19:50.747 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:50.747 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:50.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:50.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:50.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=52, tag_tid=14, entry_tid=15700' 2026-03-24T02:19:50.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:50.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:50.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=15700 2026-03-24T02:19:50.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 15700 2026-03-24T02:19:50.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:50.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:50.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:50.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:50.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:50.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:50.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:50.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:50.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:50.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:50.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:50.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:50.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:50.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:50.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:50.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:50.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:50.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:50.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:50.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:50.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:50.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:50.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:50.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:50.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:50.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:51.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:51.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:51.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:51.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:51.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:51.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:51.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:51.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:51.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:51.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:51.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:51.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:51.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:51.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:51.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:51.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:51.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:51.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:51.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=55, tag_tid=14, entry_tid=15907], [object_number=54, tag_tid=14, entry_tid=15906], [object_number=53, tag_tid=14, entry_tid=15905], [object_number=52, tag_tid=14, entry_tid=15904]]], state=connected] 2026-03-24T02:19:51.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:51.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:51.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:51.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:51.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:51.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:51.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:51.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:51.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:51.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=55, tag_tid=14, entry_tid=15907], [object_number=54, tag_tid=14, entry_tid=15906], [object_number=53, tag_tid=14, entry_tid=15905], [object_number=52, tag_tid=14, entry_tid=15904]]], state=connected] 2026-03-24T02:19:51.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=55, tag_tid=14, entry_tid=15907' 2026-03-24T02:19:51.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=55, tag_tid=14, entry_tid=15907' 2026-03-24T02:19:51.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=55, tag_tid=14, entry_tid=15907' '!=' 'object_number=52, tag_tid=14, entry_tid=15700' 2026-03-24T02:19:51.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:51.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=55, tag_tid=14, entry_tid=15907' = 'object_number=52, tag_tid=14, entry_tid=15700' 2026-03-24T02:19:51.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=55, tag_tid=14, entry_tid=15907' 2026-03-24T02:19:51.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:51.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:51.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:51.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:51.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=55, tag_tid=14, entry_tid=15907' 2026-03-24T02:19:51.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:51.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:51.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:51.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:51.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:51.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:51.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:51.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=55, tag_tid=14, entry_tid=15907' 2026-03-24T02:19:51.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:51.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:51.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=15907 2026-03-24T02:19:51.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 15907 2026-03-24T02:19:51.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:51.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:51.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:51.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:51.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:51.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:51.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:51.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:51.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:51.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:51.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:51.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:51.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:51.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:51.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:51.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:51.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:51.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:51.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:51.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:51.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:51.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:51.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:51.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:51.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:51.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:51.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:51.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:51.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:51.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:51.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:51.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:51.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:51.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:51.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:51.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:51.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:51.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:51.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:51.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:51.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:51.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:51.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:51.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:51.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:51.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:51.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:51.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:51.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:51.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:51.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:51.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:51.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:51.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:51.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:51.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:51.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:51.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:51.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:51.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:51.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:51.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:51.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:51.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:51.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:51.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:51.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:51.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:51.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:51.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:51.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:51.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:51.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:51.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:51.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=55, tag_tid=14, entry_tid=16091], [object_number=54, tag_tid=14, entry_tid=16090], [object_number=53, tag_tid=14, entry_tid=16089], [object_number=52, tag_tid=14, entry_tid=16088]]], state=connected] 2026-03-24T02:19:51.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:51.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:51.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:51.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:51.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:51.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:51.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:51.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:51.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:51.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:51.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:51.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:51.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:51.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:51.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:51.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:51.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:51.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:51.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=55, tag_tid=14, entry_tid=16091], [object_number=54, tag_tid=14, entry_tid=16090], [object_number=53, tag_tid=14, entry_tid=16089], [object_number=52, tag_tid=14, entry_tid=16088]]], state=connected] 2026-03-24T02:19:51.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=55, tag_tid=14, entry_tid=16091' 2026-03-24T02:19:51.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=55, tag_tid=14, entry_tid=16091' 2026-03-24T02:19:51.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=55, tag_tid=14, entry_tid=16091' '!=' 'object_number=55, tag_tid=14, entry_tid=15907' 2026-03-24T02:19:51.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:51.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=55, tag_tid=14, entry_tid=16091' = 'object_number=55, tag_tid=14, entry_tid=15907' 2026-03-24T02:19:51.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=55, tag_tid=14, entry_tid=16091' 2026-03-24T02:19:51.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:51.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:51.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:51.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:51.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:51.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=55, tag_tid=14, entry_tid=16091' 2026-03-24T02:19:51.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:51.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:51.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:51.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:51.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:51.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:51.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=55, tag_tid=14, entry_tid=16091' 2026-03-24T02:19:51.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:51.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:51.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=16091 2026-03-24T02:19:51.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 16091 2026-03-24T02:19:51.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:51.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:51.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:51.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:51.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:51.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:51.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:51.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:51.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:51.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:51.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:51.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:51.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:51.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:51.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:51.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:51.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:51.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:51.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:51.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:51.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=54, tag_tid=14, entry_tid=16278], [object_number=53, tag_tid=14, entry_tid=16277], [object_number=52, tag_tid=14, entry_tid=16276], [object_number=55, tag_tid=14, entry_tid=16275]]], state=connected] 2026-03-24T02:19:51.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:51.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:51.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:51.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:51.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:51.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:51.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:51.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:51.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:51.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:51.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:51.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:51.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:51.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:51.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:51.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:51.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:51.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:51.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=54, tag_tid=14, entry_tid=16278], [object_number=53, tag_tid=14, entry_tid=16277], [object_number=52, tag_tid=14, entry_tid=16276], [object_number=55, tag_tid=14, entry_tid=16275]]], state=connected] 2026-03-24T02:19:51.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:51.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=54, tag_tid=14, entry_tid=16278' 2026-03-24T02:19:51.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=54, tag_tid=14, entry_tid=16278' 2026-03-24T02:19:51.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=54, tag_tid=14, entry_tid=16278' '!=' 'object_number=55, tag_tid=14, entry_tid=16091' 2026-03-24T02:19:51.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:51.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=54, tag_tid=14, entry_tid=16278' = 'object_number=55, tag_tid=14, entry_tid=16091' 2026-03-24T02:19:51.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=54, tag_tid=14, entry_tid=16278' 2026-03-24T02:19:51.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:51.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:51.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:51.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:51.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=54, tag_tid=14, entry_tid=16278' 2026-03-24T02:19:51.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:51.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:51.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:51.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:51.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:51.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:51.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=54, tag_tid=14, entry_tid=16278' 2026-03-24T02:19:51.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:51.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:51.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=16278 2026-03-24T02:19:51.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 16278 2026-03-24T02:19:51.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:51.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:51.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:52.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:52.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:52.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:52.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:52.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:52.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:52.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:52.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:52.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:52.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:52.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:52.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:52.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:52.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:52.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:52.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:52.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:52.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:52.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:52.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:52.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:52.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:52.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:52.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:52.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:52.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:52.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:52.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:52.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:52.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:52.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:52.150 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:52.150 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:52.150 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:52.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:52.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:52.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:52.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:52.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:52.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:52.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:52.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:52.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:52.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:52.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:52.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:52.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:52.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:52.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:52.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:52.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:52.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:52.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:52.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=54, tag_tid=14, entry_tid=16410], [object_number=53, tag_tid=14, entry_tid=16409], [object_number=52, tag_tid=14, entry_tid=16408], [object_number=55, tag_tid=14, entry_tid=16407]]], state=connected] 2026-03-24T02:19:52.280 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:52.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:52.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:52.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:52.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:52.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:52.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:52.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:52.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:52.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:52.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:52.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:52.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:52.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:52.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:52.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:52.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:52.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:52.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:52.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:52.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:52.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=54, tag_tid=14, entry_tid=16410], [object_number=53, tag_tid=14, entry_tid=16409], [object_number=52, tag_tid=14, entry_tid=16408], [object_number=55, tag_tid=14, entry_tid=16407]]], state=connected] 2026-03-24T02:19:52.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:52.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=54, tag_tid=14, entry_tid=16410' 2026-03-24T02:19:52.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=54, tag_tid=14, entry_tid=16410' 2026-03-24T02:19:52.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=54, tag_tid=14, entry_tid=16410' '!=' 'object_number=54, tag_tid=14, entry_tid=16278' 2026-03-24T02:19:52.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:52.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=54, tag_tid=14, entry_tid=16410' = 'object_number=54, tag_tid=14, entry_tid=16278' 2026-03-24T02:19:52.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=54, tag_tid=14, entry_tid=16410' 2026-03-24T02:19:52.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:52.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:52.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:52.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:52.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=54, tag_tid=14, entry_tid=16410' 2026-03-24T02:19:52.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:52.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:52.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:52.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:52.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:52.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:52.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=54, tag_tid=14, entry_tid=16410' 2026-03-24T02:19:52.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:52.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:52.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=16410 2026-03-24T02:19:52.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 16410 2026-03-24T02:19:52.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:52.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:52.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:52.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:52.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:52.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:52.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:52.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:52.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:52.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:52.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:52.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:52.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:52.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:52.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:52.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:52.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:52.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:52.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:52.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:52.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:52.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:52.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:52.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:52.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:52.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:52.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:52.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:52.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:52.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:52.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:52.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:52.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:52.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:52.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:52.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:52.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:52.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:52.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:52.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:52.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:52.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:52.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:52.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:52.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:52.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:52.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:52.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:52.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:52.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:52.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:52.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=55, tag_tid=14, entry_tid=16567], [object_number=54, tag_tid=14, entry_tid=16566], [object_number=53, tag_tid=14, entry_tid=16565], [object_number=52, tag_tid=14, entry_tid=16564]]], state=connected] 2026-03-24T02:19:52.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:52.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:52.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:52.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:52.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:52.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:52.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:52.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:52.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:52.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:52.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:52.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:52.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:52.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:52.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:52.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:52.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:52.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:52.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:52.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:52.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:52.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=55, tag_tid=14, entry_tid=16567], [object_number=54, tag_tid=14, entry_tid=16566], [object_number=53, tag_tid=14, entry_tid=16565], [object_number=52, tag_tid=14, entry_tid=16564]]], state=connected] 2026-03-24T02:19:52.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:52.731 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=55, tag_tid=14, entry_tid=16567' 2026-03-24T02:19:52.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=55, tag_tid=14, entry_tid=16567' 2026-03-24T02:19:52.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=55, tag_tid=14, entry_tid=16567' '!=' 'object_number=54, tag_tid=14, entry_tid=16410' 2026-03-24T02:19:52.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:52.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=55, tag_tid=14, entry_tid=16567' = 'object_number=54, tag_tid=14, entry_tid=16410' 2026-03-24T02:19:52.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=55, tag_tid=14, entry_tid=16567' 2026-03-24T02:19:52.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:52.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:52.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:52.733 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:52.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=55, tag_tid=14, entry_tid=16567' 2026-03-24T02:19:52.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:52.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:52.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:52.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:52.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:52.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:52.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:52.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=55, tag_tid=14, entry_tid=16567' 2026-03-24T02:19:52.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:52.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:52.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=16567 2026-03-24T02:19:52.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 16567 2026-03-24T02:19:52.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:52.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:52.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:52.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:52.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:52.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:52.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:52.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:52.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:52.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:52.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:52.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:52.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:52.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:52.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:53.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:53.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:53.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:53.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:53.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:53.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:53.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:53.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:53.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:53.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:53.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:53.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:53.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:53.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:53.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:53.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:53.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:53.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:53.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=52, tag_tid=14, entry_tid=16752], [object_number=55, tag_tid=14, entry_tid=16751], [object_number=54, tag_tid=14, entry_tid=16750], [object_number=53, tag_tid=14, entry_tid=16749]]], state=connected] 2026-03-24T02:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:53.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:53.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:53.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:53.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:53.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=52, tag_tid=14, entry_tid=16752], [object_number=55, tag_tid=14, entry_tid=16751], [object_number=54, tag_tid=14, entry_tid=16750], [object_number=53, tag_tid=14, entry_tid=16749]]], state=connected] 2026-03-24T02:19:53.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=52, tag_tid=14, entry_tid=16752' 2026-03-24T02:19:53.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=52, tag_tid=14, entry_tid=16752' 2026-03-24T02:19:53.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=52, tag_tid=14, entry_tid=16752' '!=' 'object_number=55, tag_tid=14, entry_tid=16567' 2026-03-24T02:19:53.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:53.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=52, tag_tid=14, entry_tid=16752' = 'object_number=55, tag_tid=14, entry_tid=16567' 2026-03-24T02:19:53.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=52, tag_tid=14, entry_tid=16752' 2026-03-24T02:19:53.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:53.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:53.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:53.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:53.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=52, tag_tid=14, entry_tid=16752' 2026-03-24T02:19:53.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:53.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:53.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:53.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:53.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:53.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:53.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:53.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=52, tag_tid=14, entry_tid=16752' 2026-03-24T02:19:53.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:53.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:53.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=16752 2026-03-24T02:19:53.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 16752 2026-03-24T02:19:53.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:53.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:53.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:53.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:53.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:53.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:53.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:53.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:53.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:53.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:53.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:53.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:53.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:53.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:53.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:53.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:53.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:53.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:53.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:53.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:53.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:53.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:53.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:53.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:53.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:53.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:53.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:53.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:53.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:53.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:53.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:53.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:53.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:53.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:53.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:53.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:53.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:53.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:53.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:53.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:53.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:53.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:53.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:53.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:53.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:53.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:53.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:53.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:53.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:53.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:53.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:53.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:53.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:53.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:53.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:53.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:53.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:53.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:53.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:53.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=55, tag_tid=14, entry_tid=16903], [object_number=54, tag_tid=14, entry_tid=16902], [object_number=53, tag_tid=14, entry_tid=16901], [object_number=52, tag_tid=14, entry_tid=16900]]], state=connected] 2026-03-24T02:19:53.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:53.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:53.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:53.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:53.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:53.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:53.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:53.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:53.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:53.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:53.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:53.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:53.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:53.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:53.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:53.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:53.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:53.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:53.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=55, tag_tid=14, entry_tid=16903], [object_number=54, tag_tid=14, entry_tid=16902], [object_number=53, tag_tid=14, entry_tid=16901], [object_number=52, tag_tid=14, entry_tid=16900]]], state=connected] 2026-03-24T02:19:53.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=55, tag_tid=14, entry_tid=16903' 2026-03-24T02:19:53.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=55, tag_tid=14, entry_tid=16903' 2026-03-24T02:19:53.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=55, tag_tid=14, entry_tid=16903' '!=' 'object_number=52, tag_tid=14, entry_tid=16752' 2026-03-24T02:19:53.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:53.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=55, tag_tid=14, entry_tid=16903' = 'object_number=52, tag_tid=14, entry_tid=16752' 2026-03-24T02:19:53.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=55, tag_tid=14, entry_tid=16903' 2026-03-24T02:19:53.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:53.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:53.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:53.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:53.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=55, tag_tid=14, entry_tid=16903' 2026-03-24T02:19:53.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:53.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:53.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:53.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:53.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:53.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:53.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:53.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=55, tag_tid=14, entry_tid=16903' 2026-03-24T02:19:53.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:53.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:53.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=16903 2026-03-24T02:19:53.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 16903 2026-03-24T02:19:53.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:53.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:53.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:53.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:53.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:53.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:53.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:53.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:53.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:53.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:53.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:53.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:53.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:53.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:53.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:53.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:53.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:53.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:53.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:53.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:53.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:53.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:53.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:53.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:53.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:53.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:53.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=54, tag_tid=14, entry_tid=17046], [object_number=53, tag_tid=14, entry_tid=17045], [object_number=52, tag_tid=14, entry_tid=17044], [object_number=55, tag_tid=14, entry_tid=17043]]], state=connected] 2026-03-24T02:19:53.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:53.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:53.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:53.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:53.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:53.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:53.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:53.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:53.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:53.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:53.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:53.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:53.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:53.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:53.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:53.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:53.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:53.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:53.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=54, tag_tid=14, entry_tid=17046], [object_number=53, tag_tid=14, entry_tid=17045], [object_number=52, tag_tid=14, entry_tid=17044], [object_number=55, tag_tid=14, entry_tid=17043]]], state=connected] 2026-03-24T02:19:53.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:53.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=54, tag_tid=14, entry_tid=17046' 2026-03-24T02:19:53.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=54, tag_tid=14, entry_tid=17046' 2026-03-24T02:19:53.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=54, tag_tid=14, entry_tid=17046' '!=' 'object_number=55, tag_tid=14, entry_tid=16903' 2026-03-24T02:19:53.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:53.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=54, tag_tid=14, entry_tid=17046' = 'object_number=55, tag_tid=14, entry_tid=16903' 2026-03-24T02:19:53.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=54, tag_tid=14, entry_tid=17046' 2026-03-24T02:19:53.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:53.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:53.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=54, tag_tid=14, entry_tid=17046' 2026-03-24T02:19:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:53.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:53.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:53.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:53.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:53.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:53.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=54, tag_tid=14, entry_tid=17046' 2026-03-24T02:19:53.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:53.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:53.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=17046 2026-03-24T02:19:53.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 17046 2026-03-24T02:19:53.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:53.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:53.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:54.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:54.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:54.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:54.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:54.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:54.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:54.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:54.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:54.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:54.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:54.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:54.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:54.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:54.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:54.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:54.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:54.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:54.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:54.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:54.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:54.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:54.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:54.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:54.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:54.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:54.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:54.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:54.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:54.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:54.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:54.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:54.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:54.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:54.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:54.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:54.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:54.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:54.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:54.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:54.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:54.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:54.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=54, tag_tid=14, entry_tid=17250], [object_number=53, tag_tid=14, entry_tid=17249], [object_number=52, tag_tid=14, entry_tid=17248], [object_number=55, tag_tid=14, entry_tid=17247]]], state=connected] 2026-03-24T02:19:54.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:54.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:54.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:54.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:54.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:54.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:54.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:54.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:54.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:54.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:54.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:54.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:54.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:54.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:54.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:54.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:54.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:54.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 13 2026-03-24T02:19:54.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:54.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:54.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:54.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=54, tag_tid=14, entry_tid=17250], [object_number=53, tag_tid=14, entry_tid=17249], [object_number=52, tag_tid=14, entry_tid=17248], [object_number=55, tag_tid=14, entry_tid=17247]]], state=connected] 2026-03-24T02:19:54.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:54.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=54, tag_tid=14, entry_tid=17250' 2026-03-24T02:19:54.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=54, tag_tid=14, entry_tid=17250' 2026-03-24T02:19:54.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=54, tag_tid=14, entry_tid=17250' '!=' 'object_number=54, tag_tid=14, entry_tid=17046' 2026-03-24T02:19:54.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:54.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=54, tag_tid=14, entry_tid=17250' = 'object_number=54, tag_tid=14, entry_tid=17046' 2026-03-24T02:19:54.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=54, tag_tid=14, entry_tid=17250' 2026-03-24T02:19:54.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:54.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:54.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:54.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:54.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=54, tag_tid=14, entry_tid=17250' 2026-03-24T02:19:54.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:54.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:54.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:54.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:54.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:54.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:54.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:54.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=54, tag_tid=14, entry_tid=17250' 2026-03-24T02:19:54.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:54.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:54.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=17250 2026-03-24T02:19:54.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 17250 2026-03-24T02:19:54.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:54.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:54.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:54.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:54.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:54.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:54.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:54.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:54.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:54.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:54.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:54.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:54.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:54.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:54.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:54.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:54.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:54.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:54.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:54.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:54.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:54.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:54.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:54.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:54.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:54.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:54.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:54.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:54.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:54.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:54.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:54.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:54.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:54.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:54.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:54.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:54.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:54.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:54.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:54.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:54.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:54.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:54.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:54.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:54.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:54.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:54.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:54.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:54.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:54.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:54.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:54.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:54.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:54.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:54.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:54.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:54.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:54.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:54.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:54.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:54.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:54.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:54.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:54.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:54.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:54.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:54.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=59, tag_tid=14, entry_tid=17371], [object_number=58, tag_tid=14, entry_tid=17370], [object_number=57, tag_tid=14, entry_tid=17369], [object_number=56, tag_tid=14, entry_tid=17368]]], state=connected] 2026-03-24T02:19:54.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:54.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:54.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:54.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:54.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:54.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:54.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:54.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:54.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:54.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:54.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:54.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:54.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:54.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:54.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:54.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:54.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:54.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:54.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:54.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:54.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:54.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=59, tag_tid=14, entry_tid=17371], [object_number=58, tag_tid=14, entry_tid=17370], [object_number=57, tag_tid=14, entry_tid=17369], [object_number=56, tag_tid=14, entry_tid=17368]]], state=connected] 2026-03-24T02:19:54.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=59, tag_tid=14, entry_tid=17371' 2026-03-24T02:19:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=59, tag_tid=14, entry_tid=17371' 2026-03-24T02:19:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=59, tag_tid=14, entry_tid=17371' '!=' 'object_number=54, tag_tid=14, entry_tid=17250' 2026-03-24T02:19:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=59, tag_tid=14, entry_tid=17371' = 'object_number=54, tag_tid=14, entry_tid=17250' 2026-03-24T02:19:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=59, tag_tid=14, entry_tid=17371' 2026-03-24T02:19:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:54.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:54.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=59, tag_tid=14, entry_tid=17371' 2026-03-24T02:19:54.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:54.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:54.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:54.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:54.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:54.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:54.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:54.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=59, tag_tid=14, entry_tid=17371' 2026-03-24T02:19:54.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:54.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:54.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=17371 2026-03-24T02:19:54.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 17371 2026-03-24T02:19:54.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:54.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:54.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:55.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:55.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:55.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:55.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:55.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:55.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:55.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:55.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:55.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:55.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:55.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:55.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:55.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:55.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:55.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:55.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:55.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:55.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:55.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:55.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:55.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:55.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:55.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:55.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:55.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:55.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:55.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:55.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:55.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:55.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:55.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:55.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:55.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:55.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:55.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:55.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:55.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:55.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:55.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:55.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:55.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:55.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:55.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=57, tag_tid=14, entry_tid=17529], [object_number=56, tag_tid=14, entry_tid=17528], [object_number=59, tag_tid=14, entry_tid=17527], [object_number=58, tag_tid=14, entry_tid=17526]]], state=connected] 2026-03-24T02:19:55.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:55.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:55.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:55.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:55.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:55.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:55.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:55.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:55.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:55.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:55.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:55.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:55.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=57, tag_tid=14, entry_tid=17529], [object_number=56, tag_tid=14, entry_tid=17528], [object_number=59, tag_tid=14, entry_tid=17527], [object_number=58, tag_tid=14, entry_tid=17526]]], state=connected] 2026-03-24T02:19:55.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:55.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=57, tag_tid=14, entry_tid=17529' 2026-03-24T02:19:55.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=57, tag_tid=14, entry_tid=17529' 2026-03-24T02:19:55.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=57, tag_tid=14, entry_tid=17529' '!=' 'object_number=59, tag_tid=14, entry_tid=17371' 2026-03-24T02:19:55.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:55.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=57, tag_tid=14, entry_tid=17529' = 'object_number=59, tag_tid=14, entry_tid=17371' 2026-03-24T02:19:55.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=57, tag_tid=14, entry_tid=17529' 2026-03-24T02:19:55.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:55.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:55.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:55.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:55.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=14, entry_tid=17529' 2026-03-24T02:19:55.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:55.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:55.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:55.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:55.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:55.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:55.280 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:55.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=14, entry_tid=17529' 2026-03-24T02:19:55.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:55.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=17529 2026-03-24T02:19:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 17529 2026-03-24T02:19:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:55.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:55.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:55.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:55.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:55.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:55.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:55.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:55.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:55.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:55.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:55.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:55.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:55.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:55.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:55.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:55.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:55.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:55.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:55.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:55.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:55.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:55.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:55.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:55.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:55.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:55.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:55.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:55.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:55.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:55.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:55.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:55.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:55.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:55.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:55.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:55.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:55.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:55.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:55.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:55.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:55.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:55.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:55.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:55.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:55.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:55.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:55.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:55.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:55.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:55.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:55.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:55.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:55.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:55.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:55.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=58, tag_tid=14, entry_tid=17710], [object_number=57, tag_tid=14, entry_tid=17709], [object_number=56, tag_tid=14, entry_tid=17708], [object_number=59, tag_tid=14, entry_tid=17707]]], state=connected] 2026-03-24T02:19:55.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:55.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:55.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:55.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:55.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:55.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:55.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:55.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:55.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:55.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:55.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:55.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:55.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:55.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:55.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:55.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:55.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:55.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:55.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:55.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:55.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:55.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=58, tag_tid=14, entry_tid=17710], [object_number=57, tag_tid=14, entry_tid=17709], [object_number=56, tag_tid=14, entry_tid=17708], [object_number=59, tag_tid=14, entry_tid=17707]]], state=connected] 2026-03-24T02:19:55.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:55.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=58, tag_tid=14, entry_tid=17710' 2026-03-24T02:19:55.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=58, tag_tid=14, entry_tid=17710' 2026-03-24T02:19:55.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=58, tag_tid=14, entry_tid=17710' '!=' 'object_number=57, tag_tid=14, entry_tid=17529' 2026-03-24T02:19:55.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:55.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=58, tag_tid=14, entry_tid=17710' = 'object_number=57, tag_tid=14, entry_tid=17529' 2026-03-24T02:19:55.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=58, tag_tid=14, entry_tid=17710' 2026-03-24T02:19:55.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:55.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:55.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:55.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:55.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:55.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=58, tag_tid=14, entry_tid=17710' 2026-03-24T02:19:55.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:55.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:55.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:55.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:55.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:55.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:55.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=58, tag_tid=14, entry_tid=17710' 2026-03-24T02:19:55.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:55.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:55.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=17710 2026-03-24T02:19:55.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 17710 2026-03-24T02:19:55.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:55.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:55.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:55.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:55.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:55.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:55.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:55.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:55.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:55.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:55.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:55.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:55.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:55.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:55.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:55.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:55.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:55.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:55.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:55.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:55.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:55.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:55.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:55.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:55.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:55.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:55.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:55.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:55.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:55.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:55.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:55.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:55.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:55.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:55.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:55.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:55.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:55.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:55.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:55.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:55.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:55.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:55.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:55.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:55.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:55.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:55.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:55.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:55.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:55.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:55.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:55.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:56.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:56.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:56.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:56.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:56.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:56.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:56.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:56.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:56.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:56.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:56.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:56.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:56.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:56.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:56.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:56.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:56.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:56.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:56.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=58, tag_tid=14, entry_tid=17870], [object_number=57, tag_tid=14, entry_tid=17869], [object_number=56, tag_tid=14, entry_tid=17868], [object_number=59, tag_tid=14, entry_tid=17867]]], state=connected] 2026-03-24T02:19:56.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:56.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:56.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:56.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:56.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:56.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:56.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:56.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:56.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:56.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:56.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:56.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:56.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:56.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:56.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:56.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:56.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:56.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:56.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=58, tag_tid=14, entry_tid=17870], [object_number=57, tag_tid=14, entry_tid=17869], [object_number=56, tag_tid=14, entry_tid=17868], [object_number=59, tag_tid=14, entry_tid=17867]]], state=connected] 2026-03-24T02:19:56.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=58, tag_tid=14, entry_tid=17870' 2026-03-24T02:19:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=58, tag_tid=14, entry_tid=17870' 2026-03-24T02:19:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=58, tag_tid=14, entry_tid=17870' '!=' 'object_number=58, tag_tid=14, entry_tid=17710' 2026-03-24T02:19:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=58, tag_tid=14, entry_tid=17870' = 'object_number=58, tag_tid=14, entry_tid=17710' 2026-03-24T02:19:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=58, tag_tid=14, entry_tid=17870' 2026-03-24T02:19:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:56.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:56.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=58, tag_tid=14, entry_tid=17870' 2026-03-24T02:19:56.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:56.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:56.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:56.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:56.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:56.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:56.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:56.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=58, tag_tid=14, entry_tid=17870' 2026-03-24T02:19:56.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:56.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:56.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=17870 2026-03-24T02:19:56.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 17870 2026-03-24T02:19:56.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:56.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:56.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:56.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:56.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:56.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:56.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:56.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:56.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:56.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:56.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:56.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:56.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:56.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:56.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:56.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:56.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:56.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:56.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:56.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:56.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:56.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:56.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:56.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:56.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:56.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:56.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:56.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:56.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:56.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:56.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:56.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:56.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:56.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:56.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:56.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:56.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:56.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:56.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:56.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:56.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:56.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:56.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:56.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:56.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:56.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:56.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:56.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:56.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=56, tag_tid=14, entry_tid=17936], [object_number=59, tag_tid=14, entry_tid=17935], [object_number=58, tag_tid=14, entry_tid=17934], [object_number=57, tag_tid=14, entry_tid=17933]]], state=connected] 2026-03-24T02:19:56.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:56.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:56.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:56.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:56.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:56.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:56.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:56.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:56.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:56.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:56.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:56.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:56.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:56.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:56.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:56.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:56.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:56.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:56.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=56, tag_tid=14, entry_tid=17936], [object_number=59, tag_tid=14, entry_tid=17935], [object_number=58, tag_tid=14, entry_tid=17934], [object_number=57, tag_tid=14, entry_tid=17933]]], state=connected] 2026-03-24T02:19:56.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=56, tag_tid=14, entry_tid=17936' 2026-03-24T02:19:56.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=56, tag_tid=14, entry_tid=17936' 2026-03-24T02:19:56.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=56, tag_tid=14, entry_tid=17936' '!=' 'object_number=58, tag_tid=14, entry_tid=17870' 2026-03-24T02:19:56.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:56.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=56, tag_tid=14, entry_tid=17936' = 'object_number=58, tag_tid=14, entry_tid=17870' 2026-03-24T02:19:56.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=56, tag_tid=14, entry_tid=17936' 2026-03-24T02:19:56.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:56.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:56.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:56.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:56.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=56, tag_tid=14, entry_tid=17936' 2026-03-24T02:19:56.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:56.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:56.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:56.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:56.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:56.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:56.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:56.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=56, tag_tid=14, entry_tid=17936' 2026-03-24T02:19:56.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:56.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:56.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=17936 2026-03-24T02:19:56.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 17936 2026-03-24T02:19:56.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:56.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:56.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:56.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:56.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:56.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:56.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:56.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:56.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:56.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:56.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:56.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:56.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:56.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:56.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:56.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:56.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:56.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:56.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:56.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:56.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:56.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:56.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:56.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:56.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:56.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:56.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:56.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:56.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:56.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:56.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:56.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:56.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:56.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:56.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:56.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:56.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:56.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:56.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:56.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:56.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:56.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:56.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:56.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:56.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:56.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:56.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:56.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:56.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:56.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:56.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:56.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=58, tag_tid=14, entry_tid=18074], [object_number=57, tag_tid=14, entry_tid=18073], [object_number=56, tag_tid=14, entry_tid=18072], [object_number=59, tag_tid=14, entry_tid=18071]]], state=connected] 2026-03-24T02:19:56.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:56.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:56.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:56.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:56.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:56.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:56.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:56.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:56.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:56.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:56.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:56.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:56.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:56.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:56.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:56.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:56.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:56.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:56.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=58, tag_tid=14, entry_tid=18074], [object_number=57, tag_tid=14, entry_tid=18073], [object_number=56, tag_tid=14, entry_tid=18072], [object_number=59, tag_tid=14, entry_tid=18071]]], state=connected] 2026-03-24T02:19:56.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:56.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=58, tag_tid=14, entry_tid=18074' 2026-03-24T02:19:56.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=58, tag_tid=14, entry_tid=18074' 2026-03-24T02:19:56.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=58, tag_tid=14, entry_tid=18074' '!=' 'object_number=56, tag_tid=14, entry_tid=17936' 2026-03-24T02:19:56.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:56.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=58, tag_tid=14, entry_tid=18074' = 'object_number=56, tag_tid=14, entry_tid=17936' 2026-03-24T02:19:56.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=58, tag_tid=14, entry_tid=18074' 2026-03-24T02:19:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:57.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:57.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=58, tag_tid=14, entry_tid=18074' 2026-03-24T02:19:57.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:57.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:57.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:57.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:57.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:57.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:57.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:57.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=58, tag_tid=14, entry_tid=18074' 2026-03-24T02:19:57.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:57.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:57.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=18074 2026-03-24T02:19:57.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 18074 2026-03-24T02:19:57.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:57.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:57.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:57.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:57.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:57.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:57.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:57.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:57.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:57.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:57.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:57.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:57.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:57.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:57.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:57.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:57.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:57.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:57.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:57.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:57.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:57.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:57.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:57.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:57.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:57.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:57.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:57.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:57.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:57.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:57.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:57.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:57.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:57.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:57.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:57.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:57.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:57.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:57.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:57.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:57.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:57.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:57.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:57.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:57.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:57.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:57.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:57.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:57.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:57.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:57.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:57.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:57.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:57.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:57.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:57.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:57.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:57.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:57.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:57.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:57.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:57.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:57.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:57.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:57.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:57.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:57.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:57.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:57.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:57.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:57.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:57.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:57.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:57.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:57.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:57.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:57.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=59, tag_tid=14, entry_tid=18195], [object_number=58, tag_tid=14, entry_tid=18194], [object_number=57, tag_tid=14, entry_tid=18193], [object_number=56, tag_tid=14, entry_tid=18192]]], state=connected] 2026-03-24T02:19:57.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:57.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:57.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:57.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:57.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:57.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:57.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:57.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:57.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:57.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:57.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:57.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:57.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:57.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:57.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:57.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:57.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:57.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:57.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:57.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:57.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:57.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=59, tag_tid=14, entry_tid=18195], [object_number=58, tag_tid=14, entry_tid=18194], [object_number=57, tag_tid=14, entry_tid=18193], [object_number=56, tag_tid=14, entry_tid=18192]]], state=connected] 2026-03-24T02:19:57.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:57.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=59, tag_tid=14, entry_tid=18195' 2026-03-24T02:19:57.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=59, tag_tid=14, entry_tid=18195' 2026-03-24T02:19:57.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=59, tag_tid=14, entry_tid=18195' '!=' 'object_number=58, tag_tid=14, entry_tid=18074' 2026-03-24T02:19:57.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:57.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=59, tag_tid=14, entry_tid=18195' = 'object_number=58, tag_tid=14, entry_tid=18074' 2026-03-24T02:19:57.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=59, tag_tid=14, entry_tid=18195' 2026-03-24T02:19:57.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:57.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:57.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:57.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:57.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=59, tag_tid=14, entry_tid=18195' 2026-03-24T02:19:57.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:57.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:57.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:57.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=59, tag_tid=14, entry_tid=18195' 2026-03-24T02:19:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:57.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=18195 2026-03-24T02:19:57.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 18195 2026-03-24T02:19:57.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:57.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:57.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:57.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:57.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:57.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:57.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:57.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:57.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:57.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:57.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:57.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:57.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:57.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:57.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:57.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:57.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:57.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:57.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:57.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:57.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:57.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:57.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:57.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:57.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:57.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:57.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:57.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:57.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:57.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:57.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:57.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:57.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:57.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:57.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:57.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:57.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:57.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:57.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:58.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:58.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:58.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:58.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:58.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=56, tag_tid=14, entry_tid=18424], [object_number=59, tag_tid=14, entry_tid=18423], [object_number=58, tag_tid=14, entry_tid=18422], [object_number=57, tag_tid=14, entry_tid=18421]]], state=connected] 2026-03-24T02:19:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:58.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:58.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:58.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:58.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:58.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:58.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:58.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:58.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:58.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:58.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:58.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:58.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:58.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:58.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:58.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:58.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:58.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:58.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:58.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:58.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:58.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=56, tag_tid=14, entry_tid=18424], [object_number=59, tag_tid=14, entry_tid=18423], [object_number=58, tag_tid=14, entry_tid=18422], [object_number=57, tag_tid=14, entry_tid=18421]]], state=connected] 2026-03-24T02:19:58.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:58.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=56, tag_tid=14, entry_tid=18424' 2026-03-24T02:19:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=56, tag_tid=14, entry_tid=18424' 2026-03-24T02:19:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=56, tag_tid=14, entry_tid=18424' '!=' 'object_number=59, tag_tid=14, entry_tid=18195' 2026-03-24T02:19:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=56, tag_tid=14, entry_tid=18424' = 'object_number=59, tag_tid=14, entry_tid=18195' 2026-03-24T02:19:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=56, tag_tid=14, entry_tid=18424' 2026-03-24T02:19:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:58.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:58.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=56, tag_tid=14, entry_tid=18424' 2026-03-24T02:19:58.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:58.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:58.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:58.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:58.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:58.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:58.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:58.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=56, tag_tid=14, entry_tid=18424' 2026-03-24T02:19:58.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:58.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:58.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=18424 2026-03-24T02:19:58.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 18424 2026-03-24T02:19:58.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:58.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:58.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:58.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:58.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:58.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:58.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:58.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:58.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:58.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:58.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:58.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:58.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:58.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:58.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:58.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:58.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:58.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:58.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:58.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:58.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:58.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:58.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:58.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:58.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:58.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:58.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:58.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:58.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=57, tag_tid=14, entry_tid=18449], [object_number=56, tag_tid=14, entry_tid=18448], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:58.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:58.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:58.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:58.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:58.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:58.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:58.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:58.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:58.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:58.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:58.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:58.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:58.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:58.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:58.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:58.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:58.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:58.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:58.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:58.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:58.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:58.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=57, tag_tid=14, entry_tid=18449], [object_number=56, tag_tid=14, entry_tid=18448], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:58.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=57, tag_tid=14, entry_tid=18449' 2026-03-24T02:19:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=57, tag_tid=14, entry_tid=18449' 2026-03-24T02:19:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=57, tag_tid=14, entry_tid=18449' '!=' 'object_number=56, tag_tid=14, entry_tid=18424' 2026-03-24T02:19:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:19:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=57, tag_tid=14, entry_tid=18449' = 'object_number=56, tag_tid=14, entry_tid=18424' 2026-03-24T02:19:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=57, tag_tid=14, entry_tid=18449' 2026-03-24T02:19:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:58.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=15 2026-03-24T02:19:58.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=14, entry_tid=18449' 2026-03-24T02:19:58.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:19:58.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:58.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=14 2026-03-24T02:19:58.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:58.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:58.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:58.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:19:58.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=57, tag_tid=14, entry_tid=18449' 2026-03-24T02:19:58.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:19:58.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:19:58.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=18449 2026-03-24T02:19:58.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 15 = 14 -a 1 -le 18449 2026-03-24T02:19:58.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:19:58.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:58.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:19:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:58.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:58.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:58.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:58.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:58.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:58.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:58.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:58.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:58.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:58.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:58.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:58.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:58.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:58.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:58.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:58.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:58.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:58.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:58.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:58.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:58.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:58.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:58.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:58.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:58.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:58.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:58.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:58.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:58.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:58.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:58.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=57, tag_tid=14, entry_tid=18449], [object_number=56, tag_tid=14, entry_tid=18448], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:58.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:58.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:58.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:58.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:58.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:58.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:58.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:58.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:58.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:58.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:58.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:58.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:58.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:58.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:58.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:58.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:58.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:58.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:58.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:58.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:58.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:58.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=57, tag_tid=14, entry_tid=18449], [object_number=56, tag_tid=14, entry_tid=18448], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:58.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:58.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=57, tag_tid=14, entry_tid=18449' 2026-03-24T02:19:58.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=57, tag_tid=14, entry_tid=18449' 2026-03-24T02:19:58.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=57, tag_tid=14, entry_tid=18449' '!=' 'object_number=57, tag_tid=14, entry_tid=18449' 2026-03-24T02:19:58.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:58.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:19:59.280 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:19:59.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:59.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:19:59.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:19:59.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:19:59.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:19:59.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:19:59.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:19:59.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:19:59.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:59.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:19:59.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:19:59.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:19:59.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:19:59.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:19:59.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:19:59.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:19:59.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:19:59.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:19:59.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:19:59.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:19:59.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:19:59.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:19:59.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:19:59.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:19:59.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:19:59.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:19:59.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:19:59.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:19:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:59.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:59.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:19:59.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:19:59.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:19:59.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:19:59.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:59.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:59.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:59.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:19:59.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:59.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:59.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:59.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:19:59.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:59.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:59.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:59.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:59.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:59.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:59.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:59.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:59.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:59.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:59.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=57, tag_tid=14, entry_tid=18449], [object_number=56, tag_tid=14, entry_tid=18448], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:59.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:59.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:19:59.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:19:59.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:59.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:59.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:59.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:19:59.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:19:59.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:19:59.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:19:59.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:19:59.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:19:59.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:19:59.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:19:59.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:59.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:19:59.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:19:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:19:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:19:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=57, tag_tid=14, entry_tid=18449], [object_number=56, tag_tid=14, entry_tid=18448], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:19:59.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:19:59.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=57, tag_tid=14, entry_tid=18449' 2026-03-24T02:19:59.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=57, tag_tid=14, entry_tid=18449' 2026-03-24T02:19:59.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=57, tag_tid=14, entry_tid=18449' '!=' 'object_number=57, tag_tid=14, entry_tid=18449' 2026-03-24T02:19:59.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:19:59.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:20:00.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:20:00.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:00.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:00.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:00.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:20:00.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:20:00.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:20:00.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:20:00.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:20:00.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:20:00.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:00.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:20:00.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:20:00.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:20:00.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:20:00.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:20:00.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:20:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:20:00.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:20:00.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:00.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:00.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:00.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:20:00.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:00.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:20:00.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:00.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:00.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:00.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:20:00.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:00.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:00.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:00.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:20:00.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:20:00.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:20:00.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:20:00.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:00.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:20:00.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:00.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:20:00.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:20:00.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:20:00.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:20:00.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=57, tag_tid=14, entry_tid=18449], [object_number=56, tag_tid=14, entry_tid=18448], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:20:00.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:00.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:20:00.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:20:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:20:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:20:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:20:00.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:20:00.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:20:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:00.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:20:00.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:20:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:20:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:20:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:20:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=57, tag_tid=14, entry_tid=18449], [object_number=56, tag_tid=14, entry_tid=18448], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:20:00.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:00.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=57, tag_tid=14, entry_tid=18449' 2026-03-24T02:20:00.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=57, tag_tid=14, entry_tid=18449' 2026-03-24T02:20:00.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=57, tag_tid=14, entry_tid=18449' '!=' 'object_number=57, tag_tid=14, entry_tid=18449' 2026-03-24T02:20:00.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:20:00.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:20:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:20:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:20:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:20:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:20:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:20:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:20:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:20:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:20:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:20:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:20:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:20:02.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:20:02.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:20:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:20:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:20:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:20:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:20:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:20:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:20:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:20:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:20:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:20:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:20:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:20:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:20:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:20:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:20:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:20:02.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:20:02.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:20:02.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:20:02.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:20:02.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:20:02.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:20:02.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:20:02.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:20:02.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:20:02.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:20:02.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:02.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:20:02.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:20:02.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:20:02.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:20:02.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:20:02.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:02.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:02.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:02.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:20:02.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:02.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:20:02.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:02.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:02.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:02.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:20:02.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:02.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:02.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:02.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:20:02.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:20:02.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:20:02.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:20:02.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:02.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:20:02.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:02.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:20:02.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:20:02.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:20:02.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:20:02.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=57, tag_tid=14, entry_tid=18449], [object_number=56, tag_tid=14, entry_tid=18448], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:20:02.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:02.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:20:02.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:20:02.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:02.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:02.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:02.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:20:02.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:02.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:02.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:02.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:20:02.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:20:02.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:20:02.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:20:02.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:02.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:20:02.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:02.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:20:02.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:20:02.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:20:02.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:20:02.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=57, tag_tid=14, entry_tid=18449], [object_number=56, tag_tid=14, entry_tid=18448], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:20:02.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:02.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=57, tag_tid=14, entry_tid=18449' 2026-03-24T02:20:02.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=57, tag_tid=14, entry_tid=18449' 2026-03-24T02:20:02.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=57, tag_tid=14, entry_tid=18449' '!=' 'object_number=57, tag_tid=14, entry_tid=18449' 2026-03-24T02:20:02.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:20:02.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:20:04.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:20:04.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:04.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:04.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:04.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:20:04.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:20:04.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:20:04.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:20:04.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:20:04.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:20:04.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:04.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:20:04.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:20:04.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:20:04.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:20:04.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:20:04.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:20:04.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:20:04.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:20:04.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:20:04.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:20:04.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:20:04.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:20:04.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:20:04.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:04.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:20:04.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:20:04.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:20:04.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:20:04.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:20:04.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:04.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:04.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:04.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:20:04.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:04.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:20:04.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:04.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:04.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:04.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:20:04.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:04.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:04.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:04.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:20:04.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:20:04.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:20:04.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:20:04.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:04.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:20:04.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:04.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:20:04.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:20:04.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:20:04.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:20:04.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:20:04.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:04.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:20:04.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:20:04.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:04.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:04.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:04.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:20:04.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:04.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:04.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:04.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:20:04.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:20:04.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:20:04.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:20:04.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:04.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:20:04.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:04.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 14 2026-03-24T02:20:04.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 14 2026-03-24T02:20:04.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:20:04.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:20:04.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=57, tag_tid=15, entry_tid=1], [object_number=56, tag_tid=15, entry_tid=0], [object_number=59, tag_tid=14, entry_tid=18447], [object_number=58, tag_tid=14, entry_tid=18446]]], state=connected] 2026-03-24T02:20:04.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:04.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:20:04.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=57, tag_tid=15, entry_tid=1' -a 'object_number=57, tag_tid=15, entry_tid=1' = 'object_number=57, tag_tid=15, entry_tid=1' 2026-03-24T02:20:04.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:04.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror test snap5 2026-03-24T02:20:04.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:04.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:04.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:04.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap5 2026-03-24T02:20:04.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:20:04.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:20:04.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:20:05.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/test@snap5 2026-03-24T02:20:05.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'test': 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 9 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c9689d5e36 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c9689d5e36 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:15:23 2026 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:18:12 2026 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:19:29 2026 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c9689d5e36 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:20:05.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:20:05.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:05.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n snap4-clean ']' 2026-03-24T02:20:05.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap4-clean 2026-03-24T02:20:05.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:05.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:05.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap4-clean 2026-03-24T02:20:05.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:20:05.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:20:05.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:05.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:05.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:20:06.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:20:06.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:06.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:20:06.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:07.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:07.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:07.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap5 2026-03-24T02:20:07.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:07.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:07.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap5 2026-03-24T02:20:07.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:20:07.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:20:07.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:07.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:07.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:20:08.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:20:08.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:09.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:20:09.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:09.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:09.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:09.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ clean_snap_name=snap5-clean 2026-03-24T02:20:09.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap5-clean 2026-03-24T02:20:09.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:20:09.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:09.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:09.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap5-clean 2026-03-24T02:20:09.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5-clean --debug-rbd=20 --debug-journaler=20 2026-03-24T02:20:11.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:20:11.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror test 2026-03-24T02:20:11.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:20:11.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:11.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:11.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:20:11.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=8 2026-03-24T02:20:11.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:20:11.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-03-24T02:20:14.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 57492480 bytes 2026-03-24T02:20:18.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 136818688 bytes 2026-03-24T02:20:19.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:20:19.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:20:19.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:20:19.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:19.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-24T02:20:19.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:19.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:19.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:19.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:20:19.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:20:19.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:20:19.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:20:19.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:20:20.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-24T02:20:20.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:20.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:20.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:20.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:20:20.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:20:20.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:20:20.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:20:20.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:20:20.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-24T02:20:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='test: 2026-03-24T02:20:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: global_id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:20:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+replaying 2026-03-24T02:20:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: replaying, {"bytes_per_second":6499507.47,"entries_behind_primary":2543,"entries_per_second":400.3,"non_primary_position":{"entry_tid":15907,"object_number":55,"tag_tid":14},"primary_position":{"entry_tid":1,"object_number":57,"tag_tid":15},"seconds_until_synced":6} 2026-03-24T02:20:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: service: mirror.0 on vm07 2026-03-24T02:20:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:19:51 2026-03-24T02:20:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: peer_sites: 2026-03-24T02:20:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: name: cluster2 2026-03-24T02:20:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+stopped 2026-03-24T02:20:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: local image is primary 2026-03-24T02:20:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:20:04' 2026-03-24T02:20:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ state: .*up+replaying' 2026-03-24T02:20:20.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: state: up+replaying 2026-03-24T02:20:20.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ description: .*' 2026-03-24T02:20:20.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: description: replaying, {"bytes_per_second":6499507.47,"entries_behind_primary":2543,"entries_per_second":400.3,"non_primary_position":{"entry_tid":15907,"object_number":55,"tag_tid":14},"primary_position":{"entry_tid":1,"object_number":57,"tag_tid":15},"seconds_until_synced":6} 2026-03-24T02:20:20.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:20:20.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:20:20.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^up+' 2026-03-24T02:20:20.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:20:20.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep 'service: *mirror..* on ' 2026-03-24T02:20:20.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: service: mirror.0 on vm07 2026-03-24T02:20:20.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update 2026-03-24T02:20:20.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-24T02:20:20.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:19:51' 2026-03-24T02:20:20.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-24 02:19:51' 2026-03-24T02:20:20.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:20.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:20.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:20.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:20:20.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'prev_last_update=2026-03-24 02:19:51' 2026-03-24T02:20:20.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:20:20.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:20:20.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-24T02:20:20.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='OKOKOK142254303mirror.2vm0720.2.0-712-g70f8415bfalseOK42264305mirror.3vm0720.2.0-712-g70f8415bfalseOK42944297mirror.0vm0720.2.0-712-g70f8415btrueOK43554358mirror.1vm0720.2.0-712-g70f8415bfalseOKtest2d4a20c9-f6e9-4ce8-9d64-84febc265629up+replayingreplaying, {"bytes_per_second":6499507.47,"entries_behind_primary":2543,"entries_per_second":400.3,"non_primary_position":{"entry_tid":15907,"object_number":55,"tag_tid":14},"primary_position":{"entry_tid":1,"object_number":57,"tag_tid":15},"seconds_until_synced":6}42944297mirror.0vm072026-03-24 02:19:51cluster26c644ee3-7db1-4d97-9283-48b1f8a82096up+stoppedlocal image is primary2026-03-24 02:20:04' 2026-03-24T02:20:20.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update state 2026-03-24T02:20:20.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-24T02:20:20.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:19:51' 2026-03-24T02:20:20.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-24T02:20:20.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=up+replaying 2026-03-24T02:20:20.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:20:20.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep up+replaying 2026-03-24T02:20:20.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:20:20.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:20.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:20.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap6 2026-03-24T02:20:20.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap6 2026-03-24T02:20:20.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:20:20.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:20.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:20.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap6 2026-03-24T02:20:20.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:20:21.947 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6576939588916767) > weight (0.25) 2026-03-24T02:20:21.947 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5047101084333743) > weight (0.25) 2026-03-24T02:20:21.947 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.4845455389765434) > weight (0.25) 2026-03-24T02:20:21.947 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.7658187616773446) > weight (0.25) 2026-03-24T02:20:21.947 INFO:tasks.rbd_mirror_thrash:waiting for 100.4723742897092 secs before thrashing 2026-03-24T02:20:49.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-24T02:20:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-24T02:20:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:20:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:20:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:20:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:20:50.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-24T02:20:50.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:50.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:50.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:50.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:20:50.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:20:50.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:20:50.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-24T02:20:50.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:20:50.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:20:50.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:20:50.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:20:50.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:20:50.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:20:50.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:20:50.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:20:50.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:20:50.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:20:50.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:50.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:20:50.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:20:50.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:20:50.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:20:50.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:20:50.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:50.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:50.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:50.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-24T02:20:51.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:20:51.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:20:51.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:20:51.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:20:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:20:51.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:20:51.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:51.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:51.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:51.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:20:51.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:20:51.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:20:51.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:20:51.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:20:51.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:20:51.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:51.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:20:51.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:20:51.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:20:51.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:20:51.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:20:51.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:20:51.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:20:51.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:20:51.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:20:51.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:20:51.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:20:51.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:20:51.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:20:51.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:20:51.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:20:51.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:20:51.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:20:51.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:51.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:20:51.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:20:51.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:20:51.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:20:51.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:20:51.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:20:51.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:20:51.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:20:51.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:20:51.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:20:51.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:20:51.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:20:51.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:20:51.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:20:51.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:51.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:20:51.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:20:51.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:20:51.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:20:51.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:20:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:20:51.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:51.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:20:51.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:51.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:51.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:51.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:20:51.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:51.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:51.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:51.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:20:51.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:20:51.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:20:51.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:20:51.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:51.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:20:51.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:51.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 16 2026-03-24T02:20:51.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 16 2026-03-24T02:20:51.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:20:51.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=18, entry_tid=1], [object_number=64, tag_tid=18, entry_tid=0], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:51.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=65, tag_tid=17, entry_tid=8521], [object_number=64, tag_tid=17, entry_tid=8520], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:51.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=65, tag_tid=18, entry_tid=1' 2026-03-24T02:20:51.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:20:51.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:51.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:51.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:51.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:20:51.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:51.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:51.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:51.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:20:51.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:20:51.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:20:51.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:20:51.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:51.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:20:51.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:51.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 16 2026-03-24T02:20:51.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 16 2026-03-24T02:20:51.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:20:51.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=18, entry_tid=1], [object_number=64, tag_tid=18, entry_tid=0], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:51.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=65, tag_tid=17, entry_tid=8521], [object_number=64, tag_tid=17, entry_tid=8520], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:51.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:51.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=65, tag_tid=17, entry_tid=8521' 2026-03-24T02:20:51.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=65, tag_tid=18, entry_tid=1' -a 'object_number=65, tag_tid=18, entry_tid=1' = 'object_number=65, tag_tid=17, entry_tid=8521' 2026-03-24T02:20:51.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=65, tag_tid=17, entry_tid=8521' '!=' '' 2026-03-24T02:20:51.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:20:51.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=65, tag_tid=17, entry_tid=8521' = '' 2026-03-24T02:20:51.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=65, tag_tid=17, entry_tid=8521' 2026-03-24T02:20:51.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=65, tag_tid=18, entry_tid=1' 2026-03-24T02:20:51.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:20:51.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:20:51.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=18 2026-03-24T02:20:51.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=65, tag_tid=17, entry_tid=8521' 2026-03-24T02:20:51.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:20:51.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:20:51.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=17 2026-03-24T02:20:51.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=65, tag_tid=18, entry_tid=1' 2026-03-24T02:20:51.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:20:51.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:20:51.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:20:51.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=65, tag_tid=17, entry_tid=8521' 2026-03-24T02:20:51.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:20:51.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:20:51.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=8521 2026-03-24T02:20:51.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 18 = 17 -a 1 -le 8521 2026-03-24T02:20:51.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:20:51.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:20:51.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:20:51.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:20:51.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:20:51.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:20:51.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:20:51.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:20:51.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:20:51.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:20:51.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:20:51.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:20:51.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:20:51.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:20:51.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:20:51.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:20:51.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:51.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:20:51.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:20:51.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:20:51.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:20:51.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:20:51.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:20:51.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:20:51.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:20:51.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:20:51.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:20:51.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:20:51.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:20:51.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:20:51.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:20:51.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:51.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:20:51.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:20:51.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:20:51.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:20:51.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:20:51.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:51.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:51.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:51.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:20:51.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:51.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:20:51.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:51.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:51.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:51.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:20:51.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:51.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:51.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:51.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:20:51.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:20:51.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:20:51.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:20:51.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:51.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:20:51.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:51.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 16 2026-03-24T02:20:51.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 16 2026-03-24T02:20:51.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:20:51.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=18, entry_tid=1], [object_number=64, tag_tid=18, entry_tid=0], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:51.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=65, tag_tid=17, entry_tid=8521], [object_number=64, tag_tid=17, entry_tid=8520], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:51.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:51.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=65, tag_tid=18, entry_tid=1' 2026-03-24T02:20:51.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:20:51.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:51.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:51.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:51.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:20:51.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:51.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:51.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:51.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:20:51.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:20:51.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:20:51.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:20:51.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:51.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:20:51.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:51.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 16 2026-03-24T02:20:51.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 16 2026-03-24T02:20:51.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:20:51.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=18, entry_tid=1], [object_number=64, tag_tid=18, entry_tid=0], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:51.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=65, tag_tid=17, entry_tid=8521], [object_number=64, tag_tid=17, entry_tid=8520], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:51.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:51.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=65, tag_tid=17, entry_tid=8521' 2026-03-24T02:20:51.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=65, tag_tid=18, entry_tid=1' -a 'object_number=65, tag_tid=18, entry_tid=1' = 'object_number=65, tag_tid=17, entry_tid=8521' 2026-03-24T02:20:51.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=65, tag_tid=17, entry_tid=8521' '!=' 'object_number=65, tag_tid=17, entry_tid=8521' 2026-03-24T02:20:51.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:20:51.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:20:52.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:20:52.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:52.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:52.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:52.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:20:52.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:20:52.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:20:52.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:20:52.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:20:52.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:20:52.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:52.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:20:52.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:20:52.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:20:52.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:20:52.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:20:52.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:20:52.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:20:52.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:20:52.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:20:52.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:20:52.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:20:52.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:20:52.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:20:52.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:20:52.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:52.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:20:52.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:20:52.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:20:52.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:20:52.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:20:52.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:52.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:52.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:52.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:20:52.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:52.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:20:52.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:52.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:52.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:52.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:20:52.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:20:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:20:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:20:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:20:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:20:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:52.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 16 2026-03-24T02:20:52.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 16 2026-03-24T02:20:52.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:20:52.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=18, entry_tid=1], [object_number=64, tag_tid=18, entry_tid=0], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:52.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=65, tag_tid=17, entry_tid=8521], [object_number=64, tag_tid=17, entry_tid=8520], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:52.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:52.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=65, tag_tid=18, entry_tid=1' 2026-03-24T02:20:52.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:20:52.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:52.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:52.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:52.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:20:52.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:52.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:52.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:52.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:20:52.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:20:52.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:20:52.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:20:52.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:52.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:20:52.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:52.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 16 2026-03-24T02:20:52.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 16 2026-03-24T02:20:52.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:20:52.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=18, entry_tid=1], [object_number=64, tag_tid=18, entry_tid=0], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:52.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=65, tag_tid=17, entry_tid=8521], [object_number=64, tag_tid=17, entry_tid=8520], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:52.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:52.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=65, tag_tid=17, entry_tid=8521' 2026-03-24T02:20:52.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=65, tag_tid=18, entry_tid=1' -a 'object_number=65, tag_tid=18, entry_tid=1' = 'object_number=65, tag_tid=17, entry_tid=8521' 2026-03-24T02:20:52.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=65, tag_tid=17, entry_tid=8521' '!=' 'object_number=65, tag_tid=17, entry_tid=8521' 2026-03-24T02:20:52.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:20:52.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:20:53.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:20:53.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:53.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:53.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:53.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:20:53.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:20:53.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:20:53.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:20:53.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:20:53.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:20:53.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:53.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:20:53.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:20:53.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:20:53.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:20:53.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:20:53.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:20:53.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:20:53.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:20:53.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:20:53.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:20:53.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:20:53.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:20:53.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:20:53.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:20:53.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:20:53.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:20:53.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:20:53.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:53.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:20:53.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:20:53.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:20:53.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:20:53.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:20:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:20:53.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:53.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:20:53.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:53.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:53.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:53.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:20:53.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:53.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:53.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:53.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:20:53.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:20:53.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:20:53.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:20:53.286 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:53.286 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:20:53.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:53.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 16 2026-03-24T02:20:53.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 16 2026-03-24T02:20:53.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:20:53.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=18, entry_tid=1], [object_number=64, tag_tid=18, entry_tid=0], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:53.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=65, tag_tid=17, entry_tid=8521], [object_number=64, tag_tid=17, entry_tid=8520], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:53.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:53.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=65, tag_tid=18, entry_tid=1' 2026-03-24T02:20:53.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:20:53.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:53.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:53.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:53.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:20:53.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:53.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:53.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:53.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:20:53.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:20:53.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:20:53.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:20:53.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:53.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:20:53.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:53.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 16 2026-03-24T02:20:53.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 16 2026-03-24T02:20:53.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:20:53.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=18, entry_tid=1], [object_number=64, tag_tid=18, entry_tid=0], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:53.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=65, tag_tid=17, entry_tid=8521], [object_number=64, tag_tid=17, entry_tid=8520], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:53.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:53.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=65, tag_tid=17, entry_tid=8521' 2026-03-24T02:20:53.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=65, tag_tid=18, entry_tid=1' -a 'object_number=65, tag_tid=18, entry_tid=1' = 'object_number=65, tag_tid=17, entry_tid=8521' 2026-03-24T02:20:53.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=65, tag_tid=17, entry_tid=8521' '!=' 'object_number=65, tag_tid=17, entry_tid=8521' 2026-03-24T02:20:53.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:20:53.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:20:53.664 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.3084875196469118) > weight (0.25) 2026-03-24T02:20:53.664 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.886629452202353) > weight (0.25) 2026-03-24T02:20:53.664 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-03-24T02:20:53.665 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-03-24T02:20:53.665 INFO:tasks.rbd_mirror_thrash:waiting for 7.180128955265208 secs before reviving daemons 2026-03-24T02:20:53.665 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:20:53.664+0000 7f086efd2640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 26452) UID: 1000 2026-03-24T02:20:54.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:20:54.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:54.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:54.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:54.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:20:54.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:20:54.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:20:54.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:20:54.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:20:54.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:20:54.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:54.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:20:54.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:20:54.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:20:54.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:20:54.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:20:54.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:20:54.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:20:54.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:20:54.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:20:54.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:20:54.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:20:54.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:20:54.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:20:54.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:20:54.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:20:54.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:20:54.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:20:54.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:54.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:20:54.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:20:54.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:20:54.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:20:54.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:20:54.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:20:54.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:20:54.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:20:54.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:20:54.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:20:54.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:20:54.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:20:54.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:20:54.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:20:54.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:20:54.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:20:54.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:20:54.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:20:54.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:20:54.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:20:54.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:54.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:54.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:20:54.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:20:55.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:55.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:20:55.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:55.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:55.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:55.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:20:55.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:55.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:55.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:55.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:20:55.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:20:55.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:20:55.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:20:55.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:55.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:20:55.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:55.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 16 2026-03-24T02:20:55.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 16 2026-03-24T02:20:55.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:20:55.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=18, entry_tid=1], [object_number=64, tag_tid=18, entry_tid=0], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:55.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=65, tag_tid=18, entry_tid=1], [object_number=64, tag_tid=18, entry_tid=0], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:55.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:55.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=65, tag_tid=18, entry_tid=1' 2026-03-24T02:20:55.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:20:55.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:55.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:55.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:55.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:20:55.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:20:55.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:20:55.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:20:55.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:20:55.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:20:55.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:20:55.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:20:55.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:55.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:20:55.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:55.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 16 2026-03-24T02:20:55.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 16 2026-03-24T02:20:55.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:20:55.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=18, entry_tid=1], [object_number=64, tag_tid=18, entry_tid=0], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:55.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=65, tag_tid=18, entry_tid=1], [object_number=64, tag_tid=18, entry_tid=0], [object_number=67, tag_tid=17, entry_tid=8519], [object_number=66, tag_tid=17, entry_tid=8518]]], state=connected] 2026-03-24T02:20:55.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=65, tag_tid=18, entry_tid=1' 2026-03-24T02:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=65, tag_tid=18, entry_tid=1' -a 'object_number=65, tag_tid=18, entry_tid=1' = 'object_number=65, tag_tid=18, entry_tid=1' 2026-03-24T02:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror test snap6 2026-03-24T02:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap6 2026-03-24T02:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:20:56.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/test@snap6 2026-03-24T02:20:56.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'test': 2026-03-24T02:20:56.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:20:56.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:20:56.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 11 2026-03-24T02:20:56.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c9689d5e36 2026-03-24T02:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c9689d5e36 2026-03-24T02:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:15:23 2026 2026-03-24T02:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:20:06 2026 2026-03-24T02:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:20:30 2026 2026-03-24T02:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c9689d5e36 2026-03-24T02:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:20:56.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:56.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n snap5-clean ']' 2026-03-24T02:20:56.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap5-clean 2026-03-24T02:20:56.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:56.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:56.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap5-clean 2026-03-24T02:20:56.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:20:56.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:20:56.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:56.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:56.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:20:56.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:20:56.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:57.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:20:57.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:57.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:58.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:20:58.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap6 2026-03-24T02:20:58.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:20:58.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:20:58.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap6 2026-03-24T02:20:58.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:20:58.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:20:58.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:58.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:58.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:20:59.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:20:59.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:20:59.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:20:59.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:21:00.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:21:00.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:21:00.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ clean_snap_name=snap6-clean 2026-03-24T02:21:00.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap6-clean 2026-03-24T02:21:00.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:21:00.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:00.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:00.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap6-clean 2026-03-24T02:21:00.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6-clean --debug-rbd=20 --debug-journaler=20 2026-03-24T02:21:00.845 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-03-24T02:21:00.845 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:21:00.845 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-03-24T02:21:00.845 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-03-24T02:21:00.845 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-03-24T02:21:00.847 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-03-24T02:21:00.847 INFO:tasks.rbd_mirror_thrash:waiting for 81.86204326083698 secs before thrashing 2026-03-24T02:21:01.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:21:01.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror test 2026-03-24T02:21:01.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:21:01.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:01.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:01.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:21:01.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=7 2026-03-24T02:21:01.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:21:01.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-03-24T02:21:04.086 INFO:tasks.ceph.cluster2.osd.1.vm07.stderr:2026-03-24T02:21:04.072+0000 7f2e0c59f640 -1 reset not still connected to 0x5602420e2f70 2026-03-24T02:21:04.086 INFO:tasks.ceph.cluster2.osd.1.vm07.stderr:2026-03-24T02:21:04.072+0000 7f2e0c59f640 -1 reset not still connected to 0x5602468968f0 2026-03-24T02:21:04.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 48745472 bytes 2026-03-24T02:21:05.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 64718848 bytes 2026-03-24T02:21:07.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 106649600 bytes 2026-03-24T02:21:08.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:21:08.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:21:08.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:21:08.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:21:08.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-24T02:21:08.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:08.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:08.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:08.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:21:08.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:21:08.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:21:08.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:21:08.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:21:09.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-24T02:21:09.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:09.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:09.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:09.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:21:09.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:21:09.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:21:09.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:21:09.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:21:09.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-24T02:21:09.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='test: 2026-03-24T02:21:09.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: global_id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:21:09.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+replaying 2026-03-24T02:21:09.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: replaying, {"bytes_per_second":4427028.67,"entries_behind_primary":0,"entries_per_second":274.33,"non_primary_position":{"entry_tid":8504,"object_number":64,"tag_tid":17},"primary_position":{"entry_tid":5334,"object_number":62,"tag_tid":17}} 2026-03-24T02:21:09.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: service: mirror.0 on vm07 2026-03-24T02:21:09.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:20:51 2026-03-24T02:21:09.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: peer_sites: 2026-03-24T02:21:09.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: name: cluster2 2026-03-24T02:21:09.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+stopped 2026-03-24T02:21:09.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: local image is primary 2026-03-24T02:21:09.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:21:04' 2026-03-24T02:21:09.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ state: .*up+replaying' 2026-03-24T02:21:09.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: state: up+replaying 2026-03-24T02:21:09.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ description: .*' 2026-03-24T02:21:09.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: description: replaying, {"bytes_per_second":4427028.67,"entries_behind_primary":0,"entries_per_second":274.33,"non_primary_position":{"entry_tid":8504,"object_number":64,"tag_tid":17},"primary_position":{"entry_tid":5334,"object_number":62,"tag_tid":17}} 2026-03-24T02:21:09.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:21:09.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:21:09.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^up+' 2026-03-24T02:21:09.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:21:09.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep 'service: *mirror..* on ' 2026-03-24T02:21:09.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: service: mirror.0 on vm07 2026-03-24T02:21:09.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update 2026-03-24T02:21:09.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-24T02:21:09.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:20:51' 2026-03-24T02:21:09.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-24 02:20:51' 2026-03-24T02:21:09.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:09.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:09.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:09.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:21:09.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'prev_last_update=2026-03-24 02:20:51' 2026-03-24T02:21:09.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:21:09.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:21:09.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-24T02:21:09.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='OKOKOK142254303mirror.2vm0720.2.0-712-g70f8415bfalseOK42264305mirror.3vm0720.2.0-712-g70f8415bfalseOK42944297mirror.0vm0720.2.0-712-g70f8415btrueOK43554358mirror.1vm0720.2.0-712-g70f8415bfalseOKtest2d4a20c9-f6e9-4ce8-9d64-84febc265629up+replayingreplaying, {"bytes_per_second":4427028.67,"entries_behind_primary":0,"entries_per_second":274.33,"non_primary_position":{"entry_tid":8504,"object_number":64,"tag_tid":17},"primary_position":{"entry_tid":5334,"object_number":62,"tag_tid":17}}42944297mirror.0vm072026-03-24 02:20:51cluster26c644ee3-7db1-4d97-9283-48b1f8a82096up+stoppedlocal image is primary2026-03-24 02:21:04' 2026-03-24T02:21:09.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update state 2026-03-24T02:21:09.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-24T02:21:09.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:20:51' 2026-03-24T02:21:09.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-24T02:21:09.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=up+replaying 2026-03-24T02:21:09.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:21:09.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep up+replaying 2026-03-24T02:21:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:21:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:21:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:21:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap7 2026-03-24T02:21:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap7 2026-03-24T02:21:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:21:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap7 2026-03-24T02:21:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:21:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-24T02:21:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-24T02:21:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:21:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:21:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:21:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:21:41.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-24T02:21:41.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:41.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:41.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:41.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:21:41.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:21:41.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:21:41.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-24T02:21:41.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:21:41.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:21:41.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:21:41.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:21:41.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:21:41.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:21:41.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:21:41.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:21:41.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:21:41.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:21:41.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:21:41.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-24T02:21:41.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:21:41.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:21:41.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:21:41.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:21:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:21:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:21:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:21:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:21:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:21:41.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:41.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:41.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:41.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-24T02:21:41.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:21:41.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:21:41.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:21:41.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:21:41.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:21:41.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:21:41.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:41.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:41.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:41.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:21:41.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:21:41.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:21:41.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:21:41.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:21:41.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:21:41.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:41.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:21:41.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:21:41.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:21:41.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:21:41.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:21:41.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:21:41.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:21:41.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:21:41.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:21:41.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:21:41.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:21:41.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:41.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:41.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:41.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:21:41.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:21:41.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:21:41.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:41.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:41.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:41.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:21:41.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:41.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:41.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:41.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:21:41.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:21:41.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:21:41.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:21:41.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:41.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:21:41.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:41.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 18 2026-03-24T02:21:41.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 18 2026-03-24T02:21:41.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:21:41.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=73, tag_tid=21, entry_tid=1], [object_number=72, tag_tid=21, entry_tid=0], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:41.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=73, tag_tid=20, entry_tid=7701], [object_number=72, tag_tid=20, entry_tid=7700], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:41.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:41.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=73, tag_tid=21, entry_tid=1' 2026-03-24T02:21:41.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:21:41.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:41.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:41.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:41.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:21:41.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:41.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:41.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:41.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:21:41.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:21:41.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:21:41.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:21:41.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:41.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:21:41.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:41.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 18 2026-03-24T02:21:41.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 18 2026-03-24T02:21:41.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:21:41.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=73, tag_tid=21, entry_tid=1], [object_number=72, tag_tid=21, entry_tid=0], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:41.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=73, tag_tid=20, entry_tid=7701], [object_number=72, tag_tid=20, entry_tid=7700], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:41.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:41.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:41.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=73, tag_tid=21, entry_tid=1' -a 'object_number=73, tag_tid=21, entry_tid=1' = 'object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:41.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=73, tag_tid=20, entry_tid=7701' '!=' '' 2026-03-24T02:21:41.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:21:41.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=73, tag_tid=20, entry_tid=7701' = '' 2026-03-24T02:21:41.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:41.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=73, tag_tid=21, entry_tid=1' 2026-03-24T02:21:41.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:21:41.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:21:41.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=21 2026-03-24T02:21:41.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:41.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:21:41.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:21:41.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=20 2026-03-24T02:21:41.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=73, tag_tid=21, entry_tid=1' 2026-03-24T02:21:41.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:21:41.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:21:41.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:21:41.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:41.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:21:41.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:21:41.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=7701 2026-03-24T02:21:41.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 21 = 20 -a 1 -le 7701 2026-03-24T02:21:41.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:21:41.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:21:41.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:21:41.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:21:41.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:41.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:41.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:41.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:21:41.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:21:41.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:21:41.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:21:41.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:21:41.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:21:41.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:41.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:21:41.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:21:41.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:21:41.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:21:41.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:21:41.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:21:41.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:21:41.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:21:41.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:21:41.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:21:41.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:21:41.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:21:41.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:21:41.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:21:41.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:21:41.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:21:41.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:21:41.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:21:41.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:21:41.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:21:41.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:21:41.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:21:41.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:41.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:41.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:41.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:21:41.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:21:41.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:21:41.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:41.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:41.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:41.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:21:41.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:41.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:41.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:41.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:21:41.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:21:41.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:21:41.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:21:41.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:41.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:21:41.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:41.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 18 2026-03-24T02:21:41.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 18 2026-03-24T02:21:41.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:21:41.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=73, tag_tid=21, entry_tid=1], [object_number=72, tag_tid=21, entry_tid=0], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:41.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=73, tag_tid=20, entry_tid=7701], [object_number=72, tag_tid=20, entry_tid=7700], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:41.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:41.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=73, tag_tid=21, entry_tid=1' 2026-03-24T02:21:41.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:21:41.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:41.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:41.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:41.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:21:41.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:41.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:41.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:41.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:21:41.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:21:41.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:21:41.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:21:41.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:41.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:21:41.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:41.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 18 2026-03-24T02:21:41.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 18 2026-03-24T02:21:41.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:21:41.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=73, tag_tid=21, entry_tid=1], [object_number=72, tag_tid=21, entry_tid=0], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:41.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=73, tag_tid=20, entry_tid=7701], [object_number=72, tag_tid=20, entry_tid=7700], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:41.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:41.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:41.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=73, tag_tid=21, entry_tid=1' -a 'object_number=73, tag_tid=21, entry_tid=1' = 'object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:41.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=73, tag_tid=20, entry_tid=7701' '!=' 'object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:41.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:21:41.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:21:42.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:21:42.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:42.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:42.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:42.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:21:42.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:21:42.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:21:42.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:21:42.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:21:42.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:21:42.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:42.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:21:42.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:21:42.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:21:42.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:21:42.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:21:42.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:21:42.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:21:42.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:21:42.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:21:42.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:21:42.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:21:42.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:21:42.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:21:42.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:21:42.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:21:42.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:21:42.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:21:42.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:21:42.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:21:42.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:42.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:42.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:42.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:21:42.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:21:42.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:21:42.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:42.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:42.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:42.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:21:42.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:42.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:42.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:42.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:21:42.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:21:42.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:21:42.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:21:42.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:42.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:21:42.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:42.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 18 2026-03-24T02:21:42.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 18 2026-03-24T02:21:42.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:21:42.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=73, tag_tid=21, entry_tid=1], [object_number=72, tag_tid=21, entry_tid=0], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:42.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=73, tag_tid=20, entry_tid=7701], [object_number=72, tag_tid=20, entry_tid=7700], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:42.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:42.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=73, tag_tid=21, entry_tid=1' 2026-03-24T02:21:42.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:21:42.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:42.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:42.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:42.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:21:42.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:42.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:42.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:42.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:21:42.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:21:42.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:21:42.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:21:42.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:42.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:21:42.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:42.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 18 2026-03-24T02:21:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 18 2026-03-24T02:21:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:21:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=73, tag_tid=21, entry_tid=1], [object_number=72, tag_tid=21, entry_tid=0], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=73, tag_tid=20, entry_tid=7701], [object_number=72, tag_tid=20, entry_tid=7700], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:42.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:42.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:42.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=73, tag_tid=21, entry_tid=1' -a 'object_number=73, tag_tid=21, entry_tid=1' = 'object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:42.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=73, tag_tid=20, entry_tid=7701' '!=' 'object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:42.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:21:42.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:21:43.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:21:43.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:43.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:43.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:43.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:21:43.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:21:43.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:21:43.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:21:43.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:21:43.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:21:43.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:43.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:21:43.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:21:43.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:21:43.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:21:43.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:21:43.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:21:43.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:21:43.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:21:43.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:21:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:21:43.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:21:43.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:21:43.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:21:43.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:21:43.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:43.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:43.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:43.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:21:43.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:21:43.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:21:43.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:43.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:43.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:43.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:21:43.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:43.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:43.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:43.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:21:43.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:21:43.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:21:43.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:21:43.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:43.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:21:43.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:43.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 18 2026-03-24T02:21:43.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 18 2026-03-24T02:21:43.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:21:43.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=73, tag_tid=21, entry_tid=1], [object_number=72, tag_tid=21, entry_tid=0], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:43.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=73, tag_tid=20, entry_tid=7701], [object_number=72, tag_tid=20, entry_tid=7700], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:43.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:43.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=73, tag_tid=21, entry_tid=1' 2026-03-24T02:21:43.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:21:43.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:43.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:43.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:43.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:21:43.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:43.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:43.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:43.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:21:43.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:21:43.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:21:43.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:21:43.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:43.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:21:43.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:43.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 18 2026-03-24T02:21:43.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 18 2026-03-24T02:21:43.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:21:43.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=73, tag_tid=21, entry_tid=1], [object_number=72, tag_tid=21, entry_tid=0], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:43.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=73, tag_tid=20, entry_tid=7701], [object_number=72, tag_tid=20, entry_tid=7700], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:43.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:43.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:43.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=73, tag_tid=21, entry_tid=1' -a 'object_number=73, tag_tid=21, entry_tid=1' = 'object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:43.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=73, tag_tid=20, entry_tid=7701' '!=' 'object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:43.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:21:43.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:21:45.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:21:45.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:45.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:45.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:45.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:21:45.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:21:45.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:21:45.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:21:45.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:21:45.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:21:45.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:45.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:21:45.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:21:45.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:21:45.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:21:45.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:21:45.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:21:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:21:45.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:21:45.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:21:45.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:21:45.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:21:45.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:21:45.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:45.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:45.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:45.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:21:45.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:21:45.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:21:45.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:45.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:45.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:45.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:21:45.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:45.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:45.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:45.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:21:45.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:21:45.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:21:45.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:21:45.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:45.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:21:45.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:45.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 18 2026-03-24T02:21:45.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 18 2026-03-24T02:21:45.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:21:45.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=73, tag_tid=21, entry_tid=1], [object_number=72, tag_tid=21, entry_tid=0], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:45.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=73, tag_tid=20, entry_tid=7701], [object_number=72, tag_tid=20, entry_tid=7700], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:45.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:45.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=73, tag_tid=21, entry_tid=1' 2026-03-24T02:21:45.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:21:45.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:45.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:45.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:45.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:21:45.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:45.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:21:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:21:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:21:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:21:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:21:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:45.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 18 2026-03-24T02:21:45.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 18 2026-03-24T02:21:45.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:21:45.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=73, tag_tid=21, entry_tid=1], [object_number=72, tag_tid=21, entry_tid=0], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:45.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=73, tag_tid=20, entry_tid=7701], [object_number=72, tag_tid=20, entry_tid=7700], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:45.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:45.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:45.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=73, tag_tid=21, entry_tid=1' -a 'object_number=73, tag_tid=21, entry_tid=1' = 'object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:45.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=73, tag_tid=20, entry_tid=7701' '!=' 'object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:45.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:21:45.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:21:47.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:21:47.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:47.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:47.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:47.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:21:47.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:21:47.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:21:47.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:21:47.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:21:47.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:21:47.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:47.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:21:47.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:21:47.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:21:47.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:21:47.320 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:21:47.320 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:21:47.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:21:47.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:21:47.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:21:47.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:21:47.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:21:47.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:21:47.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:21:47.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:21:47.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:21:47.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:21:47.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:21:47.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:21:47.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:21:47.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:21:47.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:21:47.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:21:47.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:21:47.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:21:47.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:21:47.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:21:47.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:21:47.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:21:47.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:21:47.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:21:47.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:21:47.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:21:47.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:21:47.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:21:47.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:21:47.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:21:47.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:21:47.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:21:47.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:47.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:47.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:47.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:21:47.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:21:47.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:21:47.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:47.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:47.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:47.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:21:47.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:47.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:47.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:47.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:21:47.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:21:47.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:21:47.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:21:47.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:47.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:21:47.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:47.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 18 2026-03-24T02:21:47.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 18 2026-03-24T02:21:47.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:21:47.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=73, tag_tid=21, entry_tid=1], [object_number=72, tag_tid=21, entry_tid=0], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:47.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=73, tag_tid=20, entry_tid=7701], [object_number=72, tag_tid=20, entry_tid=7700], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:47.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:47.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=73, tag_tid=21, entry_tid=1' 2026-03-24T02:21:47.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:21:47.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:47.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:47.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:47.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:21:47.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:47.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:47.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:47.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:21:47.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:21:47.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:21:47.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:21:47.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:47.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:21:47.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:47.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 18 2026-03-24T02:21:47.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 18 2026-03-24T02:21:47.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:21:47.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=73, tag_tid=21, entry_tid=1], [object_number=72, tag_tid=21, entry_tid=0], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:47.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=73, tag_tid=20, entry_tid=7701], [object_number=72, tag_tid=20, entry_tid=7700], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:47.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:47.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:47.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=73, tag_tid=21, entry_tid=1' -a 'object_number=73, tag_tid=21, entry_tid=1' = 'object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:47.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=73, tag_tid=20, entry_tid=7701' '!=' 'object_number=73, tag_tid=20, entry_tid=7701' 2026-03-24T02:21:47.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:21:47.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:21:49.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:21:49.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:49.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:49.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:49.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:21:49.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:21:49.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:21:49.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:21:49.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:21:49.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:21:49.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:49.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:21:49.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:21:49.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:21:49.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:21:49.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:21:49.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:21:49.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:21:49.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:21:49.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:21:49.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:21:49.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:21:49.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:21:49.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:21:49.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:21:49.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:21:49.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:21:49.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:21:49.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:21:49.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:21:49.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:21:49.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:49.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:49.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:21:49.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:21:49.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:21:49.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:21:49.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:49.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:49.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:49.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:21:49.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:49.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:49.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:49.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:21:49.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:21:49.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:21:49.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:21:49.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:49.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:21:49.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:49.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 18 2026-03-24T02:21:49.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 18 2026-03-24T02:21:49.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:21:49.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=73, tag_tid=21, entry_tid=1], [object_number=72, tag_tid=21, entry_tid=0], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:49.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=73, tag_tid=21, entry_tid=1], [object_number=72, tag_tid=21, entry_tid=0], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:49.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:49.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=73, tag_tid=21, entry_tid=1' 2026-03-24T02:21:49.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:21:49.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:49.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:49.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:49.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:21:49.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:21:49.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:21:49.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:21:49.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:21:49.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:21:49.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:21:49.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:21:49.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:49.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:21:49.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:49.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 18 2026-03-24T02:21:49.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 18 2026-03-24T02:21:49.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:21:49.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=73, tag_tid=21, entry_tid=1], [object_number=72, tag_tid=21, entry_tid=0], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:49.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=73, tag_tid=21, entry_tid=1], [object_number=72, tag_tid=21, entry_tid=0], [object_number=75, tag_tid=20, entry_tid=7699], [object_number=74, tag_tid=20, entry_tid=7698]]], state=connected] 2026-03-24T02:21:49.693 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:21:49.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=73, tag_tid=21, entry_tid=1' 2026-03-24T02:21:49.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=73, tag_tid=21, entry_tid=1' -a 'object_number=73, tag_tid=21, entry_tid=1' = 'object_number=73, tag_tid=21, entry_tid=1' 2026-03-24T02:21:49.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:21:49.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror test snap7 2026-03-24T02:21:49.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:21:49.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:49.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:49.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap7 2026-03-24T02:21:49.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:21:49.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:21:49.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:21:50.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/test@snap7 2026-03-24T02:21:50.731 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'test': 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 13 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c9689d5e36 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c9689d5e36 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:15:23 2026 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:20:06 2026 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:20:30 2026 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c9689d5e36 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n snap6-clean ']' 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap6-clean 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap6-clean 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:21:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:21:51.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:21:51.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:21:52.175 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:21:52.165+0000 7fc14c2dd640 -1 bdev(0x56417b647400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-03-24T02:21:52.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:21:52.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:21:52.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:21:52.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:21:52.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap7 2026-03-24T02:21:52.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:52.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:52.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap7 2026-03-24T02:21:52.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:21:52.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:21:52.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:21:52.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:21:52.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:21:53.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:21:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:21:54.395 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:21:54.389+0000 7fc14badc640 -1 bdev(0x56417b647400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-03-24T02:21:54.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:21:54.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:21:54.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:21:54.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:21:54.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ clean_snap_name=snap7-clean 2026-03-24T02:21:54.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap7-clean 2026-03-24T02:21:54.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:21:54.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:54.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:54.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap7-clean 2026-03-24T02:21:54.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7-clean --debug-rbd=20 --debug-journaler=20 2026-03-24T02:21:55.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:21:55.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror test 2026-03-24T02:21:55.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:21:55.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:21:55.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:21:55.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:21:55.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=9 2026-03-24T02:21:55.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:21:55.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-03-24T02:21:55.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 3833856 bytes 2026-03-24T02:22:01.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 81800192 bytes 2026-03-24T02:22:02.420 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8381440284507458) > weight (0.25) 2026-03-24T02:22:02.420 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5197637944857392) > weight (0.25) 2026-03-24T02:22:02.420 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-03-24T02:22:02.420 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-03-24T02:22:02.421 INFO:tasks.rbd_mirror_thrash:waiting for 2.526589452197363 secs before reviving daemons 2026-03-24T02:22:02.423 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:22:02.420+0000 7fe8ac908640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 26421) UID: 1000 2026-03-24T02:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-24T02:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:22:04.948 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-03-24T02:22:04.948 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:22:04.948 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-03-24T02:22:04.948 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-03-24T02:22:04.948 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-03-24T02:22:04.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-03-24T02:22:04.950 INFO:tasks.rbd_mirror_thrash:waiting for 102.40085455538357 secs before thrashing 2026-03-24T02:22:05.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-24T02:22:05.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:05.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:05.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:05.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:22:05.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:22:05.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:22:05.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:22:05.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:22:05.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-24T02:22:05.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='test: 2026-03-24T02:22:05.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: global_id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:22:05.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+replaying 2026-03-24T02:22:05.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: replaying, {"bytes_per_second":1527282.23,"entries_behind_primary":0,"entries_per_second":94.09999999999999,"non_primary_position":{"entry_tid":1,"object_number":73,"tag_tid":21},"primary_position":{"entry_tid":1,"object_number":73,"tag_tid":21}} 2026-03-24T02:22:05.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: service: mirror.0 on vm07 2026-03-24T02:22:05.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:21:51 2026-03-24T02:22:05.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: peer_sites: 2026-03-24T02:22:05.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: name: cluster2 2026-03-24T02:22:05.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+stopped 2026-03-24T02:22:05.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: local image is primary 2026-03-24T02:22:05.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:22:04' 2026-03-24T02:22:05.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ state: .*up+replaying' 2026-03-24T02:22:05.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: state: up+replaying 2026-03-24T02:22:05.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ description: .*' 2026-03-24T02:22:05.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: description: replaying, {"bytes_per_second":1527282.23,"entries_behind_primary":0,"entries_per_second":94.09999999999999,"non_primary_position":{"entry_tid":1,"object_number":73,"tag_tid":21},"primary_position":{"entry_tid":1,"object_number":73,"tag_tid":21}} 2026-03-24T02:22:05.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:22:05.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:22:05.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^up+' 2026-03-24T02:22:05.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:22:05.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep 'service: *mirror..* on ' 2026-03-24T02:22:05.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: service: mirror.0 on vm07 2026-03-24T02:22:05.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update 2026-03-24T02:22:05.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-24T02:22:05.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:21:51' 2026-03-24T02:22:05.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-24 02:21:51' 2026-03-24T02:22:05.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:05.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:05.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:05.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:22:05.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'prev_last_update=2026-03-24 02:21:51' 2026-03-24T02:22:05.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:22:05.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:22:05.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-24T02:22:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='OKOKOK142264305mirror.3vm0720.2.0-712-g70f8415bfalseOK42944297mirror.0vm0720.2.0-712-g70f8415btrueOK43554358mirror.1vm0720.2.0-712-g70f8415bfalseOKtest2d4a20c9-f6e9-4ce8-9d64-84febc265629up+replayingreplaying, {"bytes_per_second":1527282.23,"entries_behind_primary":0,"entries_per_second":94.09999999999999,"non_primary_position":{"entry_tid":1,"object_number":73,"tag_tid":21},"primary_position":{"entry_tid":1,"object_number":73,"tag_tid":21}}42944297mirror.0vm072026-03-24 02:21:51cluster26c644ee3-7db1-4d97-9283-48b1f8a82096up+stoppedlocal image is primary2026-03-24 02:22:04' 2026-03-24T02:22:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update state 2026-03-24T02:22:05.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-24T02:22:05.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:21:51' 2026-03-24T02:22:05.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-24T02:22:05.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=up+replaying 2026-03-24T02:22:05.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:22:05.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep up+replaying 2026-03-24T02:22:05.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:22:05.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:05.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:05.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap8 2026-03-24T02:22:05.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap8 2026-03-24T02:22:05.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:22:05.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:05.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:05.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap8 2026-03-24T02:22:05.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:22:22.709 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-03-24T02:22:22.710 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-03-24T02:22:22.710 INFO:tasks.rbd_mirror_thrash:waiting for 1.7275870465158427 secs before reviving daemons 2026-03-24T02:22:22.710 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:22:22.708+0000 7ffbd03e2640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 28310) UID: 1000 2026-03-24T02:22:24.438 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-03-24T02:22:24.439 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:22:24.439 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-03-24T02:22:24.439 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-03-24T02:22:24.439 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-03-24T02:22:24.440 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-03-24T02:22:24.440 INFO:tasks.rbd_mirror_thrash:waiting for 104.9021977859569 secs before thrashing 2026-03-24T02:22:33.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-24T02:22:33.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:33.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:33.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:33.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-24T02:22:33.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:33.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:33.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:33.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:22:33.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:22:33.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:33.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:22:34.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-24T02:22:34.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:34.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:34.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:34.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:22:34.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:22:34.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:22:34.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-24T02:22:34.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:22:34.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:22:34.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:22:34.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:34.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:22:34.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:22:34.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:34.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:22:34.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:34.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:22:34.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:34.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-24T02:22:34.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:22:34.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:22:34.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:34.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:34.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:34.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:22:34.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:34.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:34.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:34.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:34.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:22:34.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:22:34.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:34.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:34.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:34.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:22:34.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:22:34.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:22:34.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:34.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:22:34.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:22:34.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:22:34.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:22:34.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:34.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:34.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:22:34.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:22:34.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:22:34.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:22:34.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:34.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:34.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:34.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:34.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-24T02:22:34.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:22:34.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:22:34.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:22:34.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:22:34.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:22:34.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:22:34.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:22:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:22:35.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:22:35.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:35.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:35.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:35.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:22:35.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:22:35.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:22:35.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:22:35.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:22:35.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:35.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:35.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:22:35.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:22:35.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:35.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:22:35.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:35.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:22:35.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:35.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:22:35.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:22:35.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:35.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:35.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:35.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:35.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:22:35.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:35.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:22:35.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:35.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:35.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:35.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:22:35.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:35.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:35.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:35.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:22:35.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:35.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:35.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:35.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:35.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:35.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:35.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:35.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:35.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:35.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:35.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=80, tag_tid=23, entry_tid=6348], [object_number=83, tag_tid=23, entry_tid=6347], [object_number=82, tag_tid=23, entry_tid=6346], [object_number=81, tag_tid=23, entry_tid=6345]]], state=connected] 2026-03-24T02:22:35.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:35.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:35.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:22:35.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:35.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:35.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:35.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:22:35.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:35.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:35.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:35.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:22:35.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:35.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:35.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:35.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:35.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:35.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=80, tag_tid=23, entry_tid=6348], [object_number=83, tag_tid=23, entry_tid=6347], [object_number=82, tag_tid=23, entry_tid=6346], [object_number=81, tag_tid=23, entry_tid=6345]]], state=connected] 2026-03-24T02:22:35.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:35.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=80, tag_tid=23, entry_tid=6348' 2026-03-24T02:22:35.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=81, tag_tid=24, entry_tid=1' -a 'object_number=81, tag_tid=24, entry_tid=1' = 'object_number=80, tag_tid=23, entry_tid=6348' 2026-03-24T02:22:35.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=80, tag_tid=23, entry_tid=6348' '!=' '' 2026-03-24T02:22:35.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:22:35.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=80, tag_tid=23, entry_tid=6348' = '' 2026-03-24T02:22:35.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=80, tag_tid=23, entry_tid=6348' 2026-03-24T02:22:35.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:35.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:35.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:35.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=24 2026-03-24T02:22:35.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=80, tag_tid=23, entry_tid=6348' 2026-03-24T02:22:35.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:35.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:35.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=23 2026-03-24T02:22:35.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:35.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:35.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:35.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:22:35.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=80, tag_tid=23, entry_tid=6348' 2026-03-24T02:22:35.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:35.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:35.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=6348 2026-03-24T02:22:35.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 24 = 23 -a 1 -le 6348 2026-03-24T02:22:35.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:22:35.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:22:35.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:22:35.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:22:35.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:35.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:35.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:35.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:22:35.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:22:35.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:22:35.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:22:35.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:22:35.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:35.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:35.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:22:35.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:22:35.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:35.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:22:35.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:35.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:35.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:35.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:35.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:35.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:22:35.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:35.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:35.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:35.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:35.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:22:35.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:22:35.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:35.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:35.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:35.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:35.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:35.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:35.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:22:35.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:35.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:22:35.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:35.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:35.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:35.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:22:35.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:35.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:35.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:35.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:22:35.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:35.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:35.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:35.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:35.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:35.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=80, tag_tid=23, entry_tid=6524], [object_number=83, tag_tid=23, entry_tid=6523], [object_number=82, tag_tid=23, entry_tid=6522], [object_number=81, tag_tid=23, entry_tid=6521]]], state=connected] 2026-03-24T02:22:35.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:35.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:35.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:22:35.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:35.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:35.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:35.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:22:35.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:35.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:35.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:35.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:22:35.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:35.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:35.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:35.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:35.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:35.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:35.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:35.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:35.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:35.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:35.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=80, tag_tid=23, entry_tid=6524], [object_number=83, tag_tid=23, entry_tid=6523], [object_number=82, tag_tid=23, entry_tid=6522], [object_number=81, tag_tid=23, entry_tid=6521]]], state=connected] 2026-03-24T02:22:35.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:35.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=80, tag_tid=23, entry_tid=6524' 2026-03-24T02:22:35.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=81, tag_tid=24, entry_tid=1' -a 'object_number=81, tag_tid=24, entry_tid=1' = 'object_number=80, tag_tid=23, entry_tid=6524' 2026-03-24T02:22:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=80, tag_tid=23, entry_tid=6524' '!=' 'object_number=80, tag_tid=23, entry_tid=6348' 2026-03-24T02:22:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:22:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=80, tag_tid=23, entry_tid=6524' = 'object_number=80, tag_tid=23, entry_tid=6348' 2026-03-24T02:22:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=80, tag_tid=23, entry_tid=6524' 2026-03-24T02:22:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:35.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=24 2026-03-24T02:22:35.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=80, tag_tid=23, entry_tid=6524' 2026-03-24T02:22:35.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:35.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:35.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=23 2026-03-24T02:22:35.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:35.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:35.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:35.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:22:35.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=80, tag_tid=23, entry_tid=6524' 2026-03-24T02:22:35.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:35.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=6524 2026-03-24T02:22:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 24 = 23 -a 1 -le 6524 2026-03-24T02:22:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:22:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:22:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:22:35.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:22:35.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:35.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:35.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:35.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:22:35.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:22:35.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:22:35.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:22:35.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:22:35.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:35.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:35.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:22:35.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:22:35.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:35.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:22:35.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:35.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:22:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:22:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:22:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:22:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:22:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:22:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:22:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:22:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:22:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:22:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:22:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:35.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:35.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:22:35.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:35.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:35.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:35.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:35.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:22:35.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:35.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:35.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:35.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:35.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:22:35.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:22:35.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:35.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:35.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:36.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:36.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:36.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:36.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:22:36.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:36.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:22:36.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:36.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:36.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:36.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:22:36.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:36.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:36.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:36.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:22:36.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:36.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:36.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:36.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:36.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:36.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:36.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:36.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:36.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:36.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:36.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=83, tag_tid=23, entry_tid=6603], [object_number=82, tag_tid=23, entry_tid=6602], [object_number=81, tag_tid=23, entry_tid=6601], [object_number=80, tag_tid=23, entry_tid=6600]]], state=connected] 2026-03-24T02:22:36.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:36.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:36.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:22:36.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:36.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:36.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:36.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:22:36.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:36.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:36.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:36.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:22:36.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:36.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:36.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:36.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:36.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:36.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:36.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:36.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:36.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:36.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:36.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=83, tag_tid=23, entry_tid=6603], [object_number=82, tag_tid=23, entry_tid=6602], [object_number=81, tag_tid=23, entry_tid=6601], [object_number=80, tag_tid=23, entry_tid=6600]]], state=connected] 2026-03-24T02:22:36.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:36.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=83, tag_tid=23, entry_tid=6603' 2026-03-24T02:22:36.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=81, tag_tid=24, entry_tid=1' -a 'object_number=81, tag_tid=24, entry_tid=1' = 'object_number=83, tag_tid=23, entry_tid=6603' 2026-03-24T02:22:36.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=83, tag_tid=23, entry_tid=6603' '!=' 'object_number=80, tag_tid=23, entry_tid=6524' 2026-03-24T02:22:36.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:22:36.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=83, tag_tid=23, entry_tid=6603' = 'object_number=80, tag_tid=23, entry_tid=6524' 2026-03-24T02:22:36.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=83, tag_tid=23, entry_tid=6603' 2026-03-24T02:22:36.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:36.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:36.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:36.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=24 2026-03-24T02:22:36.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=83, tag_tid=23, entry_tid=6603' 2026-03-24T02:22:36.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:36.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:36.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=23 2026-03-24T02:22:36.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:36.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:36.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:36.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:22:36.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=83, tag_tid=23, entry_tid=6603' 2026-03-24T02:22:36.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:36.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:36.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=6603 2026-03-24T02:22:36.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 24 = 23 -a 1 -le 6603 2026-03-24T02:22:36.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:22:36.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:22:36.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:22:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:22:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:22:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:22:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:22:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:22:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:22:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:22:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:22:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:22:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:22:36.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:36.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:22:36.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:22:36.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:36.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:22:36.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:36.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:22:36.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:22:36.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:22:36.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:22:36.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:22:36.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:36.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:22:36.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:36.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:22:36.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:36.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:36.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:22:36.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:36.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:36.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:36.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:36.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:22:36.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:36.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:36.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:36.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:36.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:22:36.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:22:36.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:36.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:36.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:36.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:36.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:36.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:36.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:22:36.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:36.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:22:36.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:36.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:36.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:36.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:22:36.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:36.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:36.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:36.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:22:36.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:36.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:36.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:36.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:36.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:36.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:36.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:36.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:36.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:36.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:36.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=82, tag_tid=23, entry_tid=6662], [object_number=81, tag_tid=23, entry_tid=6661], [object_number=80, tag_tid=23, entry_tid=6660], [object_number=83, tag_tid=23, entry_tid=6659]]], state=connected] 2026-03-24T02:22:36.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:36.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:36.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:22:36.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:36.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:36.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:36.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:22:36.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:36.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:36.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:36.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:22:36.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:36.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:36.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:36.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:36.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:36.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:36.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:36.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:36.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:36.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:36.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=82, tag_tid=23, entry_tid=6662], [object_number=81, tag_tid=23, entry_tid=6661], [object_number=80, tag_tid=23, entry_tid=6660], [object_number=83, tag_tid=23, entry_tid=6659]]], state=connected] 2026-03-24T02:22:36.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:36.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=82, tag_tid=23, entry_tid=6662' 2026-03-24T02:22:36.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=81, tag_tid=24, entry_tid=1' -a 'object_number=81, tag_tid=24, entry_tid=1' = 'object_number=82, tag_tid=23, entry_tid=6662' 2026-03-24T02:22:36.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=82, tag_tid=23, entry_tid=6662' '!=' 'object_number=83, tag_tid=23, entry_tid=6603' 2026-03-24T02:22:36.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:22:36.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=82, tag_tid=23, entry_tid=6662' = 'object_number=83, tag_tid=23, entry_tid=6603' 2026-03-24T02:22:36.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=82, tag_tid=23, entry_tid=6662' 2026-03-24T02:22:36.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:36.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:36.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:36.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=24 2026-03-24T02:22:36.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=82, tag_tid=23, entry_tid=6662' 2026-03-24T02:22:36.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:36.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:36.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=23 2026-03-24T02:22:36.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:36.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:36.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:36.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:22:36.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=82, tag_tid=23, entry_tid=6662' 2026-03-24T02:22:36.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:36.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:36.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=6662 2026-03-24T02:22:36.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 24 = 23 -a 1 -le 6662 2026-03-24T02:22:36.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:22:36.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:22:36.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:22:37.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:22:37.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:37.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:37.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:37.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:22:37.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:22:37.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:22:37.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:22:37.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:22:37.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:37.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:37.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:22:37.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:22:37.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:37.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:22:37.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:37.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:22:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:22:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:22:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:22:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:22:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:22:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:22:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:22:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:22:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:22:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:22:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:22:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:22:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:22:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:22:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:37.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:37.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:37.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:37.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:22:37.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:37.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:22:37.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:37.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:37.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:37.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:22:37.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:37.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:37.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:37.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:22:37.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:37.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:37.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:37.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:37.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:37.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:37.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:37.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:37.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:37.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:37.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=82, tag_tid=23, entry_tid=6726], [object_number=81, tag_tid=23, entry_tid=6725], [object_number=80, tag_tid=23, entry_tid=6724], [object_number=83, tag_tid=23, entry_tid=6723]]], state=connected] 2026-03-24T02:22:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:37.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:37.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:22:37.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:37.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:37.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:37.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:22:37.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:37.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:37.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:37.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:22:37.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:37.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:37.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:37.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:37.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:37.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:37.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:37.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:37.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:37.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:37.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=82, tag_tid=23, entry_tid=6726], [object_number=81, tag_tid=23, entry_tid=6725], [object_number=80, tag_tid=23, entry_tid=6724], [object_number=83, tag_tid=23, entry_tid=6723]]], state=connected] 2026-03-24T02:22:37.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:37.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=82, tag_tid=23, entry_tid=6726' 2026-03-24T02:22:37.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=81, tag_tid=24, entry_tid=1' -a 'object_number=81, tag_tid=24, entry_tid=1' = 'object_number=82, tag_tid=23, entry_tid=6726' 2026-03-24T02:22:37.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=82, tag_tid=23, entry_tid=6726' '!=' 'object_number=82, tag_tid=23, entry_tid=6662' 2026-03-24T02:22:37.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:22:37.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=82, tag_tid=23, entry_tid=6726' = 'object_number=82, tag_tid=23, entry_tid=6662' 2026-03-24T02:22:37.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=82, tag_tid=23, entry_tid=6726' 2026-03-24T02:22:37.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:37.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:37.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:37.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=24 2026-03-24T02:22:37.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=82, tag_tid=23, entry_tid=6726' 2026-03-24T02:22:37.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:37.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:37.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=23 2026-03-24T02:22:37.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:37.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:37.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:37.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:22:37.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=82, tag_tid=23, entry_tid=6726' 2026-03-24T02:22:37.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:37.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:37.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=6726 2026-03-24T02:22:37.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 24 = 23 -a 1 -le 6726 2026-03-24T02:22:37.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:22:37.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:22:37.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:22:37.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:22:37.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:37.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:37.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:37.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:22:37.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:22:37.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:22:37.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:22:37.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:22:37.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:37.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:37.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:22:37.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:22:37.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:37.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:22:37.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:37.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:22:37.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:37.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:22:37.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:22:37.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:37.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:22:37.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:37.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:22:37.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:22:37.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:22:37.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:22:37.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:22:37.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:37.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:22:37.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:37.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:22:37.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:37.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:37.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:22:37.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:37.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:37.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:37.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:37.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:22:37.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:37.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:37.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:37.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:37.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:22:37.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:22:37.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:37.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:37.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:37.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:37.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:37.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:37.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:22:37.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:37.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:37.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:37.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:37.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:37.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:37.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:37.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:37.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:37.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:37.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=83, tag_tid=23, entry_tid=6775], [object_number=82, tag_tid=23, entry_tid=6774], [object_number=81, tag_tid=23, entry_tid=6773], [object_number=80, tag_tid=23, entry_tid=6772]]], state=connected] 2026-03-24T02:22:37.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:37.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:37.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:22:37.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:37.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:37.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:37.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:22:37.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:37.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:37.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:37.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:22:37.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:37.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:37.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:37.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:37.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:37.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:37.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:37.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:37.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:37.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:37.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=83, tag_tid=23, entry_tid=6775], [object_number=82, tag_tid=23, entry_tid=6774], [object_number=81, tag_tid=23, entry_tid=6773], [object_number=80, tag_tid=23, entry_tid=6772]]], state=connected] 2026-03-24T02:22:37.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:37.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=83, tag_tid=23, entry_tid=6775' 2026-03-24T02:22:37.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=81, tag_tid=24, entry_tid=1' -a 'object_number=81, tag_tid=24, entry_tid=1' = 'object_number=83, tag_tid=23, entry_tid=6775' 2026-03-24T02:22:37.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=83, tag_tid=23, entry_tid=6775' '!=' 'object_number=82, tag_tid=23, entry_tid=6726' 2026-03-24T02:22:37.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:22:37.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=83, tag_tid=23, entry_tid=6775' = 'object_number=82, tag_tid=23, entry_tid=6726' 2026-03-24T02:22:37.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=83, tag_tid=23, entry_tid=6775' 2026-03-24T02:22:37.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:37.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:37.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:37.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=24 2026-03-24T02:22:37.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=83, tag_tid=23, entry_tid=6775' 2026-03-24T02:22:37.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:37.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:37.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=23 2026-03-24T02:22:37.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:37.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:37.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:37.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:22:37.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=83, tag_tid=23, entry_tid=6775' 2026-03-24T02:22:37.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:37.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:37.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=6775 2026-03-24T02:22:37.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 24 = 23 -a 1 -le 6775 2026-03-24T02:22:37.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:22:37.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:22:37.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:22:37.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:22:37.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:37.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:22:37.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:37.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:22:37.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:22:37.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:37.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:22:37.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:37.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:22:37.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:22:37.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:22:37.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:22:37.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:22:37.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:37.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:22:37.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:37.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:22:37.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:38.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:38.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:38.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:38.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:22:38.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:38.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:22:38.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:38.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:38.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:38.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:22:38.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:38.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:38.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:38.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:22:38.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:38.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:38.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:38.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:38.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:38.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=83, tag_tid=23, entry_tid=6855], [object_number=82, tag_tid=23, entry_tid=6854], [object_number=81, tag_tid=23, entry_tid=6853], [object_number=80, tag_tid=23, entry_tid=6852]]], state=connected] 2026-03-24T02:22:38.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:38.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:38.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:22:38.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:38.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:38.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:38.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:22:38.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:38.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:38.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:38.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:22:38.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:38.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:38.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:38.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:38.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:38.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:38.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:38.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:38.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:38.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:38.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=83, tag_tid=23, entry_tid=6855], [object_number=82, tag_tid=23, entry_tid=6854], [object_number=81, tag_tid=23, entry_tid=6853], [object_number=80, tag_tid=23, entry_tid=6852]]], state=connected] 2026-03-24T02:22:38.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:38.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=83, tag_tid=23, entry_tid=6855' 2026-03-24T02:22:38.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=81, tag_tid=24, entry_tid=1' -a 'object_number=81, tag_tid=24, entry_tid=1' = 'object_number=83, tag_tid=23, entry_tid=6855' 2026-03-24T02:22:38.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=83, tag_tid=23, entry_tid=6855' '!=' 'object_number=83, tag_tid=23, entry_tid=6775' 2026-03-24T02:22:38.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:22:38.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=83, tag_tid=23, entry_tid=6855' = 'object_number=83, tag_tid=23, entry_tid=6775' 2026-03-24T02:22:38.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=83, tag_tid=23, entry_tid=6855' 2026-03-24T02:22:38.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:38.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:38.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:38.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=24 2026-03-24T02:22:38.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=83, tag_tid=23, entry_tid=6855' 2026-03-24T02:22:38.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:38.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:38.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=23 2026-03-24T02:22:38.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:38.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:38.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:38.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:22:38.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=83, tag_tid=23, entry_tid=6855' 2026-03-24T02:22:38.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:38.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:38.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=6855 2026-03-24T02:22:38.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 24 = 23 -a 1 -le 6855 2026-03-24T02:22:38.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:22:38.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:22:38.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:22:38.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:22:38.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:38.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:38.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:38.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:22:38.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:22:38.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:22:38.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:22:38.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:22:38.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:38.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:38.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:22:38.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:22:38.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:38.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:22:38.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:38.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:38.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:22:38.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:38.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:38.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:38.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:38.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:22:38.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:38.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:38.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:38.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:38.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:22:38.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:22:38.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:38.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:38.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:38.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:38.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:38.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:38.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:22:38.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:38.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:22:38.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:38.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:38.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:38.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:22:38.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:38.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:38.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:38.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:22:38.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:38.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:38.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:38.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:38.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:38.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:38.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:38.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:38.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:38.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:38.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=83, tag_tid=23, entry_tid=6951], [object_number=82, tag_tid=23, entry_tid=6950], [object_number=81, tag_tid=23, entry_tid=6949], [object_number=80, tag_tid=23, entry_tid=6948]]], state=connected] 2026-03-24T02:22:38.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:38.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:38.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:22:38.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:38.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:38.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:38.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:22:38.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:38.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:38.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:38.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:22:38.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:38.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:38.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:38.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:38.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:38.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:38.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:38.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:38.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:38.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:38.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=83, tag_tid=23, entry_tid=6951], [object_number=82, tag_tid=23, entry_tid=6950], [object_number=81, tag_tid=23, entry_tid=6949], [object_number=80, tag_tid=23, entry_tid=6948]]], state=connected] 2026-03-24T02:22:38.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:38.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=83, tag_tid=23, entry_tid=6951' 2026-03-24T02:22:38.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=81, tag_tid=24, entry_tid=1' -a 'object_number=81, tag_tid=24, entry_tid=1' = 'object_number=83, tag_tid=23, entry_tid=6951' 2026-03-24T02:22:38.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=83, tag_tid=23, entry_tid=6951' '!=' 'object_number=83, tag_tid=23, entry_tid=6855' 2026-03-24T02:22:38.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:22:38.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=83, tag_tid=23, entry_tid=6951' = 'object_number=83, tag_tid=23, entry_tid=6855' 2026-03-24T02:22:38.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=83, tag_tid=23, entry_tid=6951' 2026-03-24T02:22:38.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:38.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:38.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:38.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=24 2026-03-24T02:22:38.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=83, tag_tid=23, entry_tid=6951' 2026-03-24T02:22:38.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:38.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:38.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=23 2026-03-24T02:22:38.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:38.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:38.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:38.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:22:38.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=83, tag_tid=23, entry_tid=6951' 2026-03-24T02:22:38.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:38.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:38.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=6951 2026-03-24T02:22:38.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 24 = 23 -a 1 -le 6951 2026-03-24T02:22:38.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:22:38.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:22:38.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:22:38.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:22:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:22:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:22:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:22:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:22:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:22:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:22:38.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:22:38.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:38.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:38.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:38.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:38.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:38.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:38.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:22:38.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:38.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:22:38.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:38.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:38.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:38.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:22:38.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:38.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:38.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:38.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:22:38.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:38.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:38.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:38.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:38.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:38.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:39.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:39.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:39.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:39.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:39.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=80, tag_tid=23, entry_tid=7032], [object_number=83, tag_tid=23, entry_tid=7031], [object_number=82, tag_tid=23, entry_tid=7030], [object_number=81, tag_tid=23, entry_tid=7029]]], state=connected] 2026-03-24T02:22:39.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:39.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:39.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:22:39.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:39.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:39.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:39.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:22:39.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:39.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:39.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:39.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:22:39.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:39.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:39.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:39.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:39.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:39.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:39.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:39.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:39.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:39.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:39.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=80, tag_tid=23, entry_tid=7032], [object_number=83, tag_tid=23, entry_tid=7031], [object_number=82, tag_tid=23, entry_tid=7030], [object_number=81, tag_tid=23, entry_tid=7029]]], state=connected] 2026-03-24T02:22:39.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:39.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=80, tag_tid=23, entry_tid=7032' 2026-03-24T02:22:39.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=81, tag_tid=24, entry_tid=1' -a 'object_number=81, tag_tid=24, entry_tid=1' = 'object_number=80, tag_tid=23, entry_tid=7032' 2026-03-24T02:22:39.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=80, tag_tid=23, entry_tid=7032' '!=' 'object_number=83, tag_tid=23, entry_tid=6951' 2026-03-24T02:22:39.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:22:39.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=80, tag_tid=23, entry_tid=7032' = 'object_number=83, tag_tid=23, entry_tid=6951' 2026-03-24T02:22:39.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=80, tag_tid=23, entry_tid=7032' 2026-03-24T02:22:39.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:39.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:39.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:39.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=24 2026-03-24T02:22:39.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=80, tag_tid=23, entry_tid=7032' 2026-03-24T02:22:39.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:39.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:39.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=23 2026-03-24T02:22:39.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:39.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:39.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:39.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:22:39.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=80, tag_tid=23, entry_tid=7032' 2026-03-24T02:22:39.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:39.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:39.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=7032 2026-03-24T02:22:39.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 24 = 23 -a 1 -le 7032 2026-03-24T02:22:39.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:22:39.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:22:39.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:22:39.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:22:39.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:39.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:39.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:39.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:22:39.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:22:39.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:22:39.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:22:39.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:22:39.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:39.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:39.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:22:39.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:22:39.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:39.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:22:39.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:39.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:22:39.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:39.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:22:39.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:22:39.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:39.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:22:39.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:39.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:22:39.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:22:39.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:22:39.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:22:39.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:22:39.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:39.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:22:39.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:39.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:22:39.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:39.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:39.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:22:39.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:39.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:39.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:39.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:39.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:22:39.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:39.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:39.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:39.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:39.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:22:39.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:22:39.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:39.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:39.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:39.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:39.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:39.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:39.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:22:39.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:39.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:22:39.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:39.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:39.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:39.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:22:39.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:39.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:39.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:39.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:22:39.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:39.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:39.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:39.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:39.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:39.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:39.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:39.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:39.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:39.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:39.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=80, tag_tid=23, entry_tid=7100], [object_number=83, tag_tid=23, entry_tid=7099], [object_number=82, tag_tid=23, entry_tid=7098], [object_number=81, tag_tid=23, entry_tid=7097]]], state=connected] 2026-03-24T02:22:39.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:39.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:39.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:22:39.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:39.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:39.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:39.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:22:39.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:39.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:39.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:39.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:22:39.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:39.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:39.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:39.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:39.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:39.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:39.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:39.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:39.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:39.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:39.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=80, tag_tid=23, entry_tid=7100], [object_number=83, tag_tid=23, entry_tid=7099], [object_number=82, tag_tid=23, entry_tid=7098], [object_number=81, tag_tid=23, entry_tid=7097]]], state=connected] 2026-03-24T02:22:39.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:39.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=80, tag_tid=23, entry_tid=7100' 2026-03-24T02:22:39.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=81, tag_tid=24, entry_tid=1' -a 'object_number=81, tag_tid=24, entry_tid=1' = 'object_number=80, tag_tid=23, entry_tid=7100' 2026-03-24T02:22:39.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=80, tag_tid=23, entry_tid=7100' '!=' 'object_number=80, tag_tid=23, entry_tid=7032' 2026-03-24T02:22:39.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:22:39.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=80, tag_tid=23, entry_tid=7100' = 'object_number=80, tag_tid=23, entry_tid=7032' 2026-03-24T02:22:39.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=80, tag_tid=23, entry_tid=7100' 2026-03-24T02:22:39.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:39.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:39.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=24 2026-03-24T02:22:39.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=80, tag_tid=23, entry_tid=7100' 2026-03-24T02:22:39.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:39.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:39.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=23 2026-03-24T02:22:39.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:39.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:39.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:39.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:22:39.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=80, tag_tid=23, entry_tid=7100' 2026-03-24T02:22:39.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:39.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:39.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=7100 2026-03-24T02:22:39.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 24 = 23 -a 1 -le 7100 2026-03-24T02:22:39.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:22:39.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:22:39.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:22:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:22:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:22:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:22:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:22:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:22:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:22:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:22:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:22:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:22:39.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:39.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:22:39.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:39.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:22:39.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:39.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:39.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:39.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:39.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:39.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:22:39.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:39.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:22:39.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:39.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:39.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:39.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:22:39.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:39.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:39.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:39.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:22:39.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:39.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:39.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:39.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:39.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:39.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:39.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:39.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:39.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:39.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:39.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=80, tag_tid=23, entry_tid=7252], [object_number=83, tag_tid=23, entry_tid=7251], [object_number=82, tag_tid=23, entry_tid=7250], [object_number=81, tag_tid=23, entry_tid=7249]]], state=connected] 2026-03-24T02:22:39.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:39.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:39.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:22:39.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:39.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:39.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:39.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:22:39.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:39.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:39.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:39.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:22:39.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:39.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:39.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:39.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:39.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:39.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:39.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:39.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:39.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:39.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:39.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=80, tag_tid=23, entry_tid=7252], [object_number=83, tag_tid=23, entry_tid=7251], [object_number=82, tag_tid=23, entry_tid=7250], [object_number=81, tag_tid=23, entry_tid=7249]]], state=connected] 2026-03-24T02:22:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:39.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=80, tag_tid=23, entry_tid=7252' 2026-03-24T02:22:39.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=81, tag_tid=24, entry_tid=1' -a 'object_number=81, tag_tid=24, entry_tid=1' = 'object_number=80, tag_tid=23, entry_tid=7252' 2026-03-24T02:22:39.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=80, tag_tid=23, entry_tid=7252' '!=' 'object_number=80, tag_tid=23, entry_tid=7100' 2026-03-24T02:22:39.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:22:39.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=80, tag_tid=23, entry_tid=7252' = 'object_number=80, tag_tid=23, entry_tid=7100' 2026-03-24T02:22:39.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=80, tag_tid=23, entry_tid=7252' 2026-03-24T02:22:39.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:39.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:39.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:39.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=24 2026-03-24T02:22:39.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=80, tag_tid=23, entry_tid=7252' 2026-03-24T02:22:39.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:39.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:39.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=23 2026-03-24T02:22:39.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:39.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:39.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:39.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:22:39.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=80, tag_tid=23, entry_tid=7252' 2026-03-24T02:22:39.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:39.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:39.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=7252 2026-03-24T02:22:39.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 24 = 23 -a 1 -le 7252 2026-03-24T02:22:39.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:22:39.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:22:39.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:22:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:22:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:22:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:22:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:22:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:22:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:22:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:22:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:22:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:22:40.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:40.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:22:40.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:40.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:22:40.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:22:40.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:40.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:22:40.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:40.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:22:40.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:22:40.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:40.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:40.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:40.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:40.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:40.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:22:40.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:40.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:22:40.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:40.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:40.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:40.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:22:40.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:40.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:40.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:40.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:22:40.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:40.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:40.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:40.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:40.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:40.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=81, tag_tid=23, entry_tid=7421], [object_number=80, tag_tid=23, entry_tid=7420], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:40.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:40.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:40.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:22:40.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:22:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:22:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:40.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:40.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:40.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:40.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:40.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:40.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:40.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:40.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:40.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=81, tag_tid=23, entry_tid=7421], [object_number=80, tag_tid=23, entry_tid=7420], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:40.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:40.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=81, tag_tid=23, entry_tid=7421' 2026-03-24T02:22:40.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=81, tag_tid=24, entry_tid=1' -a 'object_number=81, tag_tid=24, entry_tid=1' = 'object_number=81, tag_tid=23, entry_tid=7421' 2026-03-24T02:22:40.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=81, tag_tid=23, entry_tid=7421' '!=' 'object_number=80, tag_tid=23, entry_tid=7252' 2026-03-24T02:22:40.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:22:40.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=81, tag_tid=23, entry_tid=7421' = 'object_number=80, tag_tid=23, entry_tid=7252' 2026-03-24T02:22:40.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=81, tag_tid=23, entry_tid=7421' 2026-03-24T02:22:40.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:40.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:40.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:40.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=24 2026-03-24T02:22:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=23, entry_tid=7421' 2026-03-24T02:22:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:22:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:40.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=23 2026-03-24T02:22:40.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:40.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:40.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:40.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:22:40.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=81, tag_tid=23, entry_tid=7421' 2026-03-24T02:22:40.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:22:40.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:22:40.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=7421 2026-03-24T02:22:40.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 24 = 23 -a 1 -le 7421 2026-03-24T02:22:40.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:22:40.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:22:40.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:22:40.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:22:40.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:40.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:40.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:40.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:22:40.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:22:40.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:22:40.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:22:40.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:22:40.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:40.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:40.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:22:40.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:22:40.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:40.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:22:40.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:40.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:40.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:40.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:40.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:40.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:22:40.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:40.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:22:40.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:40.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:40.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:40.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:22:40.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:40.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:40.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:40.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:22:40.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:40.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:40.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:40.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:40.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:40.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:40.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:40.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:40.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:40.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:40.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=81, tag_tid=23, entry_tid=7421], [object_number=80, tag_tid=23, entry_tid=7420], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:40.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:40.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:40.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:22:40.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:40.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:40.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:40.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:22:40.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:40.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:40.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:40.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:22:40.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:40.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:40.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:40.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:40.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:40.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:40.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:40.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:40.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:40.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:40.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=81, tag_tid=23, entry_tid=7421], [object_number=80, tag_tid=23, entry_tid=7420], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:40.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:40.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=81, tag_tid=23, entry_tid=7421' 2026-03-24T02:22:40.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=81, tag_tid=24, entry_tid=1' -a 'object_number=81, tag_tid=24, entry_tid=1' = 'object_number=81, tag_tid=23, entry_tid=7421' 2026-03-24T02:22:40.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=81, tag_tid=23, entry_tid=7421' '!=' 'object_number=81, tag_tid=23, entry_tid=7421' 2026-03-24T02:22:40.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:22:40.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:22:41.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:22:41.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:41.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:41.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:41.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:22:41.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:22:41.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:22:41.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:22:41.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:22:41.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:41.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:41.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:22:41.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:22:41.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:41.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:22:41.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:41.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:22:41.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:41.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:22:41.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:22:41.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:41.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:22:41.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:41.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:22:41.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:22:41.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:22:41.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:22:41.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:22:41.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:41.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:22:41.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:41.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:22:41.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:41.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:41.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:22:41.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:41.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:41.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:41.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:41.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:22:41.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:41.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:41.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:41.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:41.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:22:41.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:22:41.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:41.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:41.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:41.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:41.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:41.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:41.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:22:41.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:41.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:22:41.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:41.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:41.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:41.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:22:41.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:41.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:41.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:41.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:22:41.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:41.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:41.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:41.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:41.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:41.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:41.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:41.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:41.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:41.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:41.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=81, tag_tid=23, entry_tid=7421], [object_number=80, tag_tid=23, entry_tid=7420], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:41.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:22:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:22:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:22:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:41.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:41.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:41.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:41.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:41.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:41.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:41.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=81, tag_tid=23, entry_tid=7421], [object_number=80, tag_tid=23, entry_tid=7420], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:41.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:41.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=81, tag_tid=23, entry_tid=7421' 2026-03-24T02:22:41.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=81, tag_tid=24, entry_tid=1' -a 'object_number=81, tag_tid=24, entry_tid=1' = 'object_number=81, tag_tid=23, entry_tid=7421' 2026-03-24T02:22:41.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=81, tag_tid=23, entry_tid=7421' '!=' 'object_number=81, tag_tid=23, entry_tid=7421' 2026-03-24T02:22:41.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:22:41.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:22:42.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:22:42.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:42.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:42.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:42.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:22:42.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:22:42.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:22:42.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:22:42.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:22:42.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:42.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:42.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:22:42.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:22:42.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:42.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:22:42.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:42.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:22:42.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:42.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:22:42.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:22:42.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:42.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:22:42.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:42.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:42.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:42.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:42.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:42.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:42.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:22:42.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:42.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:22:42.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:42.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:42.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:42.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:22:42.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:42.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:42.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:42.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:22:42.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:42.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:42.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:42.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:42.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:42.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:42.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:42.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:42.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:42.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:42.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=81, tag_tid=23, entry_tid=7421], [object_number=80, tag_tid=23, entry_tid=7420], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:42.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:42.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:42.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:22:42.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:42.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:42.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:42.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:22:42.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:42.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:42.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:42.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:22:42.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:42.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:42.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:42.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:42.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:42.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:42.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:42.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:42.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:42.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:42.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=81, tag_tid=23, entry_tid=7421], [object_number=80, tag_tid=23, entry_tid=7420], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:42.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:42.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=81, tag_tid=23, entry_tid=7421' 2026-03-24T02:22:42.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=81, tag_tid=24, entry_tid=1' -a 'object_number=81, tag_tid=24, entry_tid=1' = 'object_number=81, tag_tid=23, entry_tid=7421' 2026-03-24T02:22:42.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=81, tag_tid=23, entry_tid=7421' '!=' 'object_number=81, tag_tid=23, entry_tid=7421' 2026-03-24T02:22:42.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:22:42.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:22:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:22:43.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:43.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:22:43.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:22:43.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:43.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:22:43.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:43.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:22:43.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:22:43.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:22:43.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:22:43.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:22:43.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:43.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:22:43.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:43.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:22:43.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:43.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:43.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:43.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:43.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:43.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:22:44.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:44.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:22:44.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:44.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:44.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:44.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:22:44.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:44.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:44.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:44.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:22:44.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:44.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:44.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:44.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:44.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:44.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:44.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:44.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:44.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:44.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:44.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=81, tag_tid=23, entry_tid=7421], [object_number=80, tag_tid=23, entry_tid=7420], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:44.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:44.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:44.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:22:44.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:44.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:44.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:44.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:22:44.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:44.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:44.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:44.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:22:44.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:44.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:44.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:44.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:44.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:44.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:44.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:44.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:44.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:44.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:44.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=81, tag_tid=23, entry_tid=7421], [object_number=80, tag_tid=23, entry_tid=7420], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:44.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:44.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=81, tag_tid=23, entry_tid=7421' 2026-03-24T02:22:44.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=81, tag_tid=24, entry_tid=1' -a 'object_number=81, tag_tid=24, entry_tid=1' = 'object_number=81, tag_tid=23, entry_tid=7421' 2026-03-24T02:22:44.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=81, tag_tid=23, entry_tid=7421' '!=' 'object_number=81, tag_tid=23, entry_tid=7421' 2026-03-24T02:22:44.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:22:44.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:22:46.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:22:46.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:46.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:46.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:46.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:22:46.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:22:46.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:22:46.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:22:46.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:22:46.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:46.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:46.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:22:46.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:22:46.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:46.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:22:46.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:22:46.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:22:46.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:22:46.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:22:46.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:22:46.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:22:46.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:22:46.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:22:46.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:22:46.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:22:46.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:22:46.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:22:46.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:22:46.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:46.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:22:46.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:46.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:22:46.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:46.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:22:46.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:22:46.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:22:46.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:22:46.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:22:46.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:22:46.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:22:46.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:22:46.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:22:46.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:22:46.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:22:46.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:22:46.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:22:46.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:22:46.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:22:46.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:22:46.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:46.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:46.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:22:46.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:22:46.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:46.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:22:46.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:46.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:46.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:46.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:22:46.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:46.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:46.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:46.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:22:46.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:46.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:46.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:46.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:46.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:46.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:46.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:46.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:46.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:46.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:46.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:46.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:46.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:46.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:22:46.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:46.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:46.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:46.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:22:46.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:22:46.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:22:46.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:22:46.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:22:46.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:22:46.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:22:46.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:22:46.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:46.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:22:46.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:46.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 20 2026-03-24T02:22:46.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 20 2026-03-24T02:22:46.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:22:46.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:46.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=81, tag_tid=24, entry_tid=1], [object_number=80, tag_tid=24, entry_tid=0], [object_number=83, tag_tid=23, entry_tid=7419], [object_number=82, tag_tid=23, entry_tid=7418]]], state=connected] 2026-03-24T02:22:46.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:22:46.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:46.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=81, tag_tid=24, entry_tid=1' -a 'object_number=81, tag_tid=24, entry_tid=1' = 'object_number=81, tag_tid=24, entry_tid=1' 2026-03-24T02:22:46.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:46.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror test snap8 2026-03-24T02:22:46.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:46.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:46.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:46.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap8 2026-03-24T02:22:46.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:22:46.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:22:46.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:22:47.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/test@snap8 2026-03-24T02:22:47.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'test': 2026-03-24T02:22:47.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:22:47.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:22:47.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 15 2026-03-24T02:22:47.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c9689d5e36 2026-03-24T02:22:47.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c9689d5e36 2026-03-24T02:22:47.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:22:47.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:22:47.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:22:47.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:22:47.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:15:23 2026 2026-03-24T02:22:47.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:21:51 2026 2026-03-24T02:22:47.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:21:58 2026 2026-03-24T02:22:47.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:22:47.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c9689d5e36 2026-03-24T02:22:47.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:22:47.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:22:47.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:22:47.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:22:47.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:47.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n snap7-clean ']' 2026-03-24T02:22:47.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap7-clean 2026-03-24T02:22:47.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:47.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:47.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap7-clean 2026-03-24T02:22:47.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:22:47.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:22:47.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:22:47.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:22:47.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:22:48.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:22:48.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:22:48.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:22:48.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:22:48.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:22:49.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:49.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap8 2026-03-24T02:22:49.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:49.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:49.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap8 2026-03-24T02:22:49.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:22:49.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:22:49.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:22:49.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:22:49.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:22:49.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:22:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:22:50.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:22:50.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:22:50.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:22:51.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:51.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ clean_snap_name=snap8-clean 2026-03-24T02:22:51.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap8-clean 2026-03-24T02:22:51.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:22:51.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:51.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:51.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap8-clean 2026-03-24T02:22:51.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8-clean --debug-rbd=20 --debug-journaler=20 2026-03-24T02:22:52.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:22:52.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror test 2026-03-24T02:22:52.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:22:52.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:52.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:52.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:22:52.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=7 2026-03-24T02:22:52.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:22:52.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-03-24T02:22:54.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 32078848 bytes 2026-03-24T02:22:58.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 84845568 bytes 2026-03-24T02:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-24T02:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:23:00.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-24T02:23:00.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:00.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:00.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:00.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:23:00.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:23:00.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:23:00.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:23:00.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:23:00.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-24T02:23:00.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='test: 2026-03-24T02:23:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: global_id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:23:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+replaying 2026-03-24T02:23:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: replaying, {"bytes_per_second":2380838.5,"entries_behind_primary":0,"entries_per_second":147.43,"non_primary_position":{"entry_tid":1,"object_number":81,"tag_tid":24},"primary_position":{"entry_tid":1,"object_number":81,"tag_tid":24}} 2026-03-24T02:23:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: service: mirror.0 on vm07 2026-03-24T02:23:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:22:51 2026-03-24T02:23:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: peer_sites: 2026-03-24T02:23:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: name: cluster2 2026-03-24T02:23:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+stopped 2026-03-24T02:23:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: local image is primary 2026-03-24T02:23:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:22:34' 2026-03-24T02:23:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ state: .*up+replaying' 2026-03-24T02:23:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: state: up+replaying 2026-03-24T02:23:00.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ description: .*' 2026-03-24T02:23:00.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: description: replaying, {"bytes_per_second":2380838.5,"entries_behind_primary":0,"entries_per_second":147.43,"non_primary_position":{"entry_tid":1,"object_number":81,"tag_tid":24},"primary_position":{"entry_tid":1,"object_number":81,"tag_tid":24}} 2026-03-24T02:23:00.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:23:00.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:23:00.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^up+' 2026-03-24T02:23:00.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:23:00.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep 'service: *mirror..* on ' 2026-03-24T02:23:00.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: service: mirror.0 on vm07 2026-03-24T02:23:00.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update 2026-03-24T02:23:00.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-24T02:23:00.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:22:51' 2026-03-24T02:23:00.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-24 02:22:51' 2026-03-24T02:23:00.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:00.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:00.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:00.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:23:00.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'prev_last_update=2026-03-24 02:22:51' 2026-03-24T02:23:00.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:23:00.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:23:00.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-24T02:23:00.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='OKOKOK142264305mirror.3vm0720.2.0-712-g70f8415bfalseOK42944297mirror.0vm0720.2.0-712-g70f8415btrueOK43554358mirror.1vm0720.2.0-712-g70f8415bfalseOK44584461mirror.2vm0720.2.0-712-g70f8415bfalseOKtest2d4a20c9-f6e9-4ce8-9d64-84febc265629up+replayingreplaying, {"bytes_per_second":2380838.5,"entries_behind_primary":0,"entries_per_second":147.43,"non_primary_position":{"entry_tid":1,"object_number":81,"tag_tid":24},"primary_position":{"entry_tid":1,"object_number":81,"tag_tid":24}}42944297mirror.0vm072026-03-24 02:22:51cluster26c644ee3-7db1-4d97-9283-48b1f8a82096up+stoppedlocal image is primary2026-03-24 02:22:34' 2026-03-24T02:23:00.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update state 2026-03-24T02:23:00.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-24T02:23:00.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:22:51' 2026-03-24T02:23:00.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-24T02:23:00.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=up+replaying 2026-03-24T02:23:00.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:23:00.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep up+replaying 2026-03-24T02:23:00.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap9 2026-03-24T02:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap9 2026-03-24T02:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap9 2026-03-24T02:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:23:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-24T02:23:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-24T02:23:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:23:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:23:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:23:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:23:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-24T02:23:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:23:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:23:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:23:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-24T02:23:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:23:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:23:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:23:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:23:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:23:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:23:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:23:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:23:31.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:23:31.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:23:31.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:23:31.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-24T02:23:31.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:23:31.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:23:31.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:23:31.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:23:31.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:23:31.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:23:31.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:23:31.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:23:31.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:23:31.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:23:31.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:23:31.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:23:31.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:23:31.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:31.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:31.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:31.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-24T02:23:31.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:23:31.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:23:31.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:23:31.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:23:31.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:23:31.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:23:31.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:31.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:31.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:31.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:23:31.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:23:31.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:23:31.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:23:31.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:23:31.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:23:31.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:31.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:23:31.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:23:31.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:23:31.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:23:31.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:23:31.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:23:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:23:31.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:23:31.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:23:31.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:31.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:31.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:31.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:23:31.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:23:31.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:23:31.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:31.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:31.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:31.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:23:31.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:31.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:31.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:31.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:23:31.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:23:31.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:23:31.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:23:31.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:31.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:23:31.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:31.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 22 2026-03-24T02:23:31.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 22 2026-03-24T02:23:31.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:23:31.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=27, entry_tid=1], [object_number=88, tag_tid=27, entry_tid=0], [object_number=90, tag_tid=26, entry_tid=7038], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:31.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=90, tag_tid=26, entry_tid=7038], [object_number=89, tag_tid=26, entry_tid=7037], [object_number=88, tag_tid=26, entry_tid=7036], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:31.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:31.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=89, tag_tid=27, entry_tid=1' 2026-03-24T02:23:31.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:23:31.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:31.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:31.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:31.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:23:31.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:31.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:31.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:31.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:23:31.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:23:31.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:23:31.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:23:31.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:31.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:23:31.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:31.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 22 2026-03-24T02:23:31.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 22 2026-03-24T02:23:31.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:23:31.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=27, entry_tid=1], [object_number=88, tag_tid=27, entry_tid=0], [object_number=90, tag_tid=26, entry_tid=7038], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:31.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=90, tag_tid=26, entry_tid=7038], [object_number=89, tag_tid=26, entry_tid=7037], [object_number=88, tag_tid=26, entry_tid=7036], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:31.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:31.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:31.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=89, tag_tid=27, entry_tid=1' -a 'object_number=89, tag_tid=27, entry_tid=1' = 'object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:31.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=90, tag_tid=26, entry_tid=7038' '!=' '' 2026-03-24T02:23:31.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:23:31.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=90, tag_tid=26, entry_tid=7038' = '' 2026-03-24T02:23:31.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:31.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=89, tag_tid=27, entry_tid=1' 2026-03-24T02:23:31.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:23:31.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:23:31.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=27 2026-03-24T02:23:31.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:31.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:23:31.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:23:31.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=26 2026-03-24T02:23:31.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=89, tag_tid=27, entry_tid=1' 2026-03-24T02:23:31.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:23:31.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:23:31.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:23:31.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:31.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:23:31.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:23:31.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=7038 2026-03-24T02:23:31.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 27 = 26 -a 1 -le 7038 2026-03-24T02:23:31.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:23:31.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:23:31.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:23:31.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:23:31.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:31.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:31.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:31.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:23:31.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:23:31.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:23:31.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:23:31.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:23:31.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:23:31.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:31.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:23:31.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:23:31.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:23:31.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:23:31.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:23:31.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:23:31.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:23:31.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:23:31.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:23:31.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:23:31.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:23:31.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:23:31.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:23:31.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:23:31.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:23:31.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:23:31.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:23:31.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:23:31.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:23:31.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:23:31.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:23:31.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:23:31.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:23:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:23:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:23:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:23:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:23:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:23:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:23:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:23:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:23:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:23:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:23:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:23:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:23:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:23:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:23:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:23:31.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:31.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:31.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:31.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:23:31.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:23:31.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:23:31.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:31.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:31.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:31.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:23:31.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:31.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:31.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:31.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:23:31.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:23:31.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:23:31.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:23:31.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:31.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:23:31.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:31.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 22 2026-03-24T02:23:31.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 22 2026-03-24T02:23:31.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:23:31.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=27, entry_tid=1], [object_number=88, tag_tid=27, entry_tid=0], [object_number=90, tag_tid=26, entry_tid=7038], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:31.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=90, tag_tid=26, entry_tid=7038], [object_number=89, tag_tid=26, entry_tid=7037], [object_number=88, tag_tid=26, entry_tid=7036], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:31.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:31.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=89, tag_tid=27, entry_tid=1' 2026-03-24T02:23:31.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:23:31.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:31.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:31.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:31.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:23:31.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:31.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:31.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:31.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:23:31.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:23:31.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:23:31.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:23:31.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:31.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:23:31.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:31.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 22 2026-03-24T02:23:31.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 22 2026-03-24T02:23:31.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:23:31.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=27, entry_tid=1], [object_number=88, tag_tid=27, entry_tid=0], [object_number=90, tag_tid=26, entry_tid=7038], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:31.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=90, tag_tid=26, entry_tid=7038], [object_number=89, tag_tid=26, entry_tid=7037], [object_number=88, tag_tid=26, entry_tid=7036], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:31.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=89, tag_tid=27, entry_tid=1' -a 'object_number=89, tag_tid=27, entry_tid=1' = 'object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=90, tag_tid=26, entry_tid=7038' '!=' 'object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:23:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:23:32.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:23:32.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:32.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:32.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:32.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:23:32.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:23:32.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:23:32.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:23:32.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:23:32.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:23:32.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:32.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:23:32.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:23:32.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:23:32.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:23:32.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:23:32.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:23:32.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:23:32.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:23:32.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:23:32.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:23:32.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:23:32.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:23:32.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:23:32.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:23:32.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:23:32.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:23:32.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:23:32.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:23:32.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:23:32.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:23:32.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:32.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:32.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:32.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:23:32.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:23:32.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:23:32.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:23:32.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:32.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:23:32.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:32.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 22 2026-03-24T02:23:32.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 22 2026-03-24T02:23:32.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:23:32.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=27, entry_tid=1], [object_number=88, tag_tid=27, entry_tid=0], [object_number=90, tag_tid=26, entry_tid=7038], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:32.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=90, tag_tid=26, entry_tid=7038], [object_number=89, tag_tid=26, entry_tid=7037], [object_number=88, tag_tid=26, entry_tid=7036], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:32.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:32.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=89, tag_tid=27, entry_tid=1' 2026-03-24T02:23:32.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:23:32.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:32.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:32.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:32.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:23:32.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:32.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:32.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:32.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:23:32.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:23:32.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:23:32.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:23:32.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:32.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:23:32.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:32.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 22 2026-03-24T02:23:32.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 22 2026-03-24T02:23:32.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:23:32.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=27, entry_tid=1], [object_number=88, tag_tid=27, entry_tid=0], [object_number=90, tag_tid=26, entry_tid=7038], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:32.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=90, tag_tid=26, entry_tid=7038], [object_number=89, tag_tid=26, entry_tid=7037], [object_number=88, tag_tid=26, entry_tid=7036], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:32.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:32.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:32.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=89, tag_tid=27, entry_tid=1' -a 'object_number=89, tag_tid=27, entry_tid=1' = 'object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:32.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=90, tag_tid=26, entry_tid=7038' '!=' 'object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:32.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:23:32.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:23:33.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:23:33.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:33.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:33.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:33.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:23:33.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:23:33.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:23:33.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:23:33.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:23:33.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:23:33.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:33.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:23:33.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:23:33.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:23:33.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:23:33.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:23:33.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:23:33.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:23:33.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:23:33.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:23:33.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:23:33.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:23:33.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:23:33.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:23:33.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:23:33.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:23:33.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:23:33.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:23:33.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:23:33.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:23:33.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:23:33.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:23:33.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:23:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:23:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:23:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:23:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:23:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:23:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:23:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:23:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:23:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:23:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:23:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:23:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:23:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:23:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:23:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:23:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:23:33.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:33.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:33.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:33.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:23:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:23:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:23:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:23:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:23:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:23:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:23:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:23:33.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:33.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:23:33.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:33.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 22 2026-03-24T02:23:33.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 22 2026-03-24T02:23:33.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:23:33.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=27, entry_tid=1], [object_number=88, tag_tid=27, entry_tid=0], [object_number=90, tag_tid=26, entry_tid=7038], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:33.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=90, tag_tid=26, entry_tid=7038], [object_number=89, tag_tid=26, entry_tid=7037], [object_number=88, tag_tid=26, entry_tid=7036], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:33.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:33.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=89, tag_tid=27, entry_tid=1' 2026-03-24T02:23:33.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:23:33.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:33.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:33.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:33.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:23:33.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:33.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:33.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:33.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:23:33.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:23:33.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:23:33.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:23:33.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:33.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:23:33.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:33.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 22 2026-03-24T02:23:33.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 22 2026-03-24T02:23:33.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:23:33.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=27, entry_tid=1], [object_number=88, tag_tid=27, entry_tid=0], [object_number=90, tag_tid=26, entry_tid=7038], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:33.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=90, tag_tid=26, entry_tid=7038], [object_number=89, tag_tid=26, entry_tid=7037], [object_number=88, tag_tid=26, entry_tid=7036], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:33.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:33.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=89, tag_tid=27, entry_tid=1' -a 'object_number=89, tag_tid=27, entry_tid=1' = 'object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:33.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=90, tag_tid=26, entry_tid=7038' '!=' 'object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:33.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:23:33.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:23:35.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:23:35.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:35.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:35.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:35.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:23:35.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:23:35.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:23:35.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:23:35.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:23:35.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:23:35.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:35.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:23:35.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:23:35.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:23:35.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:23:35.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:23:35.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:23:35.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:23:35.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:23:35.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:23:35.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:23:35.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:23:35.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:23:35.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:23:35.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:23:35.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:23:35.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:23:35.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:23:35.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:23:35.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:23:35.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:23:35.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:23:35.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:23:35.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:23:35.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:23:35.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:23:35.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:23:35.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:23:35.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:23:35.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:23:35.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:23:35.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:23:35.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:23:35.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:23:35.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:23:35.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:23:35.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:23:35.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:23:35.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:23:35.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:35.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:35.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:35.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:23:35.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:23:35.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:23:35.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:35.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:35.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:35.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:23:35.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:35.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:35.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:35.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:23:35.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:23:35.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:23:35.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:23:35.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:35.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:23:35.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 22 2026-03-24T02:23:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 22 2026-03-24T02:23:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:23:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=27, entry_tid=1], [object_number=88, tag_tid=27, entry_tid=0], [object_number=90, tag_tid=26, entry_tid=7038], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=90, tag_tid=26, entry_tid=7038], [object_number=89, tag_tid=26, entry_tid=7037], [object_number=88, tag_tid=26, entry_tid=7036], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:35.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:35.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=89, tag_tid=27, entry_tid=1' 2026-03-24T02:23:35.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:23:35.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:35.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:35.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:35.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:23:35.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:35.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:35.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:35.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:23:35.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:23:35.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:23:35.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:23:35.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:35.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:23:35.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:35.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 22 2026-03-24T02:23:35.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 22 2026-03-24T02:23:35.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:23:35.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=27, entry_tid=1], [object_number=88, tag_tid=27, entry_tid=0], [object_number=90, tag_tid=26, entry_tid=7038], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:35.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=90, tag_tid=26, entry_tid=7038], [object_number=89, tag_tid=26, entry_tid=7037], [object_number=88, tag_tid=26, entry_tid=7036], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:35.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=89, tag_tid=27, entry_tid=1' -a 'object_number=89, tag_tid=27, entry_tid=1' = 'object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=90, tag_tid=26, entry_tid=7038' '!=' 'object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:23:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:23:37.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:23:37.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:37.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:37.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:37.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:23:37.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:23:37.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:23:37.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:23:37.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:23:37.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:23:37.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:37.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:23:37.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:23:37.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:23:37.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:23:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:23:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:23:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:23:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:23:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:23:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:23:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:23:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:23:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:23:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:23:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:23:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:23:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:23:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:23:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:23:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:23:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:23:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:23:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:23:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:23:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:23:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:23:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:23:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:23:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:23:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:23:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:23:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:23:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:23:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:23:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:23:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:23:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:23:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:23:37.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:37.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:37.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:37.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:23:37.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:23:37.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:23:37.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:37.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:37.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:37.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:23:37.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:37.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:37.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:37.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:23:37.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:23:37.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:23:37.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:23:37.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:37.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:23:37.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:37.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 22 2026-03-24T02:23:37.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 22 2026-03-24T02:23:37.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:23:37.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=27, entry_tid=1], [object_number=88, tag_tid=27, entry_tid=0], [object_number=90, tag_tid=26, entry_tid=7038], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:37.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=90, tag_tid=26, entry_tid=7038], [object_number=89, tag_tid=26, entry_tid=7037], [object_number=88, tag_tid=26, entry_tid=7036], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:37.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:37.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=89, tag_tid=27, entry_tid=1' 2026-03-24T02:23:37.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:23:37.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:37.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:37.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:37.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:23:37.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:37.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:37.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:37.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:23:37.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:23:37.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:23:37.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:23:37.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:37.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:23:37.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:37.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 22 2026-03-24T02:23:37.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 22 2026-03-24T02:23:37.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:23:37.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=27, entry_tid=1], [object_number=88, tag_tid=27, entry_tid=0], [object_number=90, tag_tid=26, entry_tid=7038], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:37.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=90, tag_tid=26, entry_tid=7038], [object_number=89, tag_tid=26, entry_tid=7037], [object_number=88, tag_tid=26, entry_tid=7036], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:37.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:37.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:37.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=89, tag_tid=27, entry_tid=1' -a 'object_number=89, tag_tid=27, entry_tid=1' = 'object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:37.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=90, tag_tid=26, entry_tid=7038' '!=' 'object_number=90, tag_tid=26, entry_tid=7038' 2026-03-24T02:23:37.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:23:37.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:23:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:23:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:23:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:23:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:23:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:23:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:23:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:23:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:23:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:23:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:23:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:23:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:23:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:23:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:23:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:23:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:23:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:23:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:23:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:23:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:23:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:23:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:23:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:23:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:23:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:23:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:23:39.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:23:39.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:39.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:39.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:23:39.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:23:39.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:23:39.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:23:39.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:39.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:39.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:39.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:23:39.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:39.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:39.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:39.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:23:39.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:23:39.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:23:39.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:23:39.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:39.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:39.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:23:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 22 2026-03-24T02:23:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 22 2026-03-24T02:23:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:23:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=27, entry_tid=1], [object_number=88, tag_tid=27, entry_tid=0], [object_number=90, tag_tid=26, entry_tid=7038], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=89, tag_tid=27, entry_tid=1], [object_number=88, tag_tid=27, entry_tid=0], [object_number=90, tag_tid=26, entry_tid=7038], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:39.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:39.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=89, tag_tid=27, entry_tid=1' 2026-03-24T02:23:39.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:23:39.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:39.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:39.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:39.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:23:39.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:23:39.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:23:39.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:23:39.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:23:39.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:23:39.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:23:39.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:23:39.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:39.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:23:39.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:39.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 22 2026-03-24T02:23:39.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 22 2026-03-24T02:23:39.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:23:39.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=27, entry_tid=1], [object_number=88, tag_tid=27, entry_tid=0], [object_number=90, tag_tid=26, entry_tid=7038], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:39.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=89, tag_tid=27, entry_tid=1], [object_number=88, tag_tid=27, entry_tid=0], [object_number=90, tag_tid=26, entry_tid=7038], [object_number=91, tag_tid=26, entry_tid=7035]]], state=connected] 2026-03-24T02:23:39.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:23:39.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=89, tag_tid=27, entry_tid=1' 2026-03-24T02:23:39.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=89, tag_tid=27, entry_tid=1' -a 'object_number=89, tag_tid=27, entry_tid=1' = 'object_number=89, tag_tid=27, entry_tid=1' 2026-03-24T02:23:39.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:23:39.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror test snap9 2026-03-24T02:23:39.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:39.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:39.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:39.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap9 2026-03-24T02:23:39.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:23:39.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:23:39.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:23:40.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/test@snap9 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'test': 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 17 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c9689d5e36 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c9689d5e36 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:15:23 2026 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:21:51 2026 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:22:59 2026 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c9689d5e36 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:23:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:23:40.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:23:40.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n snap8-clean ']' 2026-03-24T02:23:40.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap8-clean 2026-03-24T02:23:40.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:40.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:40.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap8-clean 2026-03-24T02:23:40.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:23:40.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:23:40.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:23:40.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:23:40.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:23:41.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:23:41.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:23:42.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:23:42.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:23:42.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:23:42.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:23:42.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap9 2026-03-24T02:23:42.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:42.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:42.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap9 2026-03-24T02:23:42.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:23:42.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:23:42.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:23:42.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:23:42.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:23:43.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:23:43.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:23:44.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:23:44.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:23:44.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:23:44.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:23:44.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ clean_snap_name=snap9-clean 2026-03-24T02:23:44.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap9-clean 2026-03-24T02:23:44.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:23:44.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:44.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:44.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap9-clean 2026-03-24T02:23:44.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9-clean --debug-rbd=20 --debug-journaler=20 2026-03-24T02:23:45.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:23:45.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror test 2026-03-24T02:23:45.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:23:45.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:45.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:45.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:23:45.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=6 2026-03-24T02:23:45.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:23:45.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-03-24T02:23:47.351 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.44566504795715034) > weight (0.25) 2026-03-24T02:23:47.352 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-03-24T02:23:47.352 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-03-24T02:23:47.352 INFO:tasks.rbd_mirror_thrash:waiting for 9.298516902904796 secs before reviving daemons 2026-03-24T02:23:47.353 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:23:47.348+0000 7ff157b91640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 29191) UID: 1000 2026-03-24T02:23:47.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 21424128 bytes 2026-03-24T02:23:49.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 56722432 bytes 2026-03-24T02:23:49.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 57324544 bytes 2026-03-24T02:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-24T02:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:23:51.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:23:52.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-24T02:23:52.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:52.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:52.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:52.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:23:52.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local description_pattern= 2026-03-24T02:23:52.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local service_pattern= 2026-03-24T02:23:52.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:23:52.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:23:52.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-24T02:23:52.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='test: 2026-03-24T02:23:52.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: global_id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:23:52.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+replaying 2026-03-24T02:23:52.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: replaying, {"bytes_per_second":405083.43,"entries_behind_primary":0,"entries_per_second":25.57,"non_primary_position":{"entry_tid":1,"object_number":89,"tag_tid":27},"primary_position":{"entry_tid":1,"object_number":89,"tag_tid":27}} 2026-03-24T02:23:52.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: service: mirror.0 on vm07 2026-03-24T02:23:52.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:23:51 2026-03-24T02:23:52.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: peer_sites: 2026-03-24T02:23:52.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: name: cluster2 2026-03-24T02:23:52.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: state: up+stopped 2026-03-24T02:23:52.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: description: local image is primary 2026-03-24T02:23:52.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: last_update: 2026-03-24 02:23:34' 2026-03-24T02:23:52.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ state: .*up+replaying' 2026-03-24T02:23:52.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: state: up+replaying 2026-03-24T02:23:52.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^ description: .*' 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: description: replaying, {"bytes_per_second":405083.43,"entries_behind_primary":0,"entries_per_second":25.57,"non_primary_position":{"entry_tid":1,"object_number":89,"tag_tid":27},"primary_position":{"entry_tid":1,"object_number":89,"tag_tid":27}} 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep '^up+' 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep 'service: *mirror..* on ' 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: service: mirror.0 on vm07 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:23:51' 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-24 02:23:51' 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state_pattern=up+replaying 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'prev_last_update=2026-03-24 02:23:51' 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ CEPH_ARGS= 2026-03-24T02:23:52.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-24T02:23:53.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status='OKOKOK142264305mirror.3vm0720.2.0-712-g70f8415bfalseOK42944297mirror.0vm0720.2.0-712-g70f8415btrueOK44584461mirror.2vm0720.2.0-712-g70f8415bfalseOKtest2d4a20c9-f6e9-4ce8-9d64-84febc265629up+replayingreplaying, {"bytes_per_second":405083.43,"entries_behind_primary":0,"entries_per_second":25.57,"non_primary_position":{"entry_tid":1,"object_number":89,"tag_tid":27},"primary_position":{"entry_tid":1,"object_number":89,"tag_tid":27}}42944297mirror.0vm072026-03-24 02:23:51cluster26c644ee3-7db1-4d97-9283-48b1f8a82096up+stoppedlocal image is primary2026-03-24 02:23:34' 2026-03-24T02:23:53.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_update state 2026-03-24T02:23:53.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-24T02:23:53.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_update='2026-03-24 02:23:51' 2026-03-24T02:23:53.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-24T02:23:53.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=up+replaying 2026-03-24T02:23:53.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo up+replaying 2026-03-24T02:23:53.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep up+replaying 2026-03-24T02:23:53.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:up+replaying 2026-03-24T02:23:53.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:23:53.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:23:53.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap10 2026-03-24T02:23:53.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap10 2026-03-24T02:23:53.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:23:53.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:23:53.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:23:53.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap10 2026-03-24T02:23:53.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:23:56.652 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-03-24T02:23:56.652 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:23:56.652 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-03-24T02:23:56.652 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-03-24T02:23:56.652 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-03-24T02:23:56.653 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-03-24T02:23:56.653 INFO:tasks.rbd_mirror_thrash:waiting for 97.86502768878617 secs before thrashing 2026-03-24T02:24:09.343 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-03-24T02:24:09.343 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-03-24T02:24:09.344 INFO:tasks.rbd_mirror_thrash:waiting for 6.306711794953186 secs before reviving daemons 2026-03-24T02:24:09.344 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:24:09.340+0000 7f8e72925640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 39809) UID: 1000 2026-03-24T02:24:15.651 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-03-24T02:24:15.651 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:24:15.651 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-03-24T02:24:15.651 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-03-24T02:24:15.652 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-03-24T02:24:15.653 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-03-24T02:24:15.653 INFO:tasks.rbd_mirror_thrash:waiting for 62.92863591497051 secs before thrashing 2026-03-24T02:24:22.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-24T02:24:22.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:22.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:22.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:22.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-24T02:24:22.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:22.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:22.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:22.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:24:22.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:24:22.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:24:22.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:24:23.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-24T02:24:23.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:23.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:23.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:23.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:24:23.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:24:23.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:24:23.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-24T02:24:23.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:24:23.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:24:23.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:24:23.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:24:23.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:24:23.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:24:23.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:24:23.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:24:23.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:24:23.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:24:23.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:24:23.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-24T02:24:23.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:24:23.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:24:23.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:24:23.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:24:23.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:24:23.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:24:23.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:24:23.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:23.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:23.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:23.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:24:23.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:24:23.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:24:23.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:24:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:24:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:24:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:24:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:24:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:24:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:24:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:24:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:24:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:24:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:24:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:24:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:24:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:24:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:24:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:24:23.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:23.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:23.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:23.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-24T02:24:23.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:24:23.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:24:23.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:24:23.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:24:23.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:24:23.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:24:23.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:24:23.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:23.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:23.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:23.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:24:23.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:24:23.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:24:23.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:24:23.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:24:23.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:23.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:23.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:24:23.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:24:23.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:24:23.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:24:23.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:24:23.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:23.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:23.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:24:23.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:24:23.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:24:23.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:24:23.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:24:24.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:24.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:24.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:24.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:24:24.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:24.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:24:24.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:24.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:24.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:24.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:24:24.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:24.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:24.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:24.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:24:24.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:24.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:24.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:24.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:24.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:24.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:24.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:24.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:24.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:24.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:24.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606], [object_number=93, tag_tid=29, entry_tid=5605], [object_number=92, tag_tid=29, entry_tid=5604]]], state=connected] 2026-03-24T02:24:24.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:24.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:24.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:24:24.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:24.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:24.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:24.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:24:24.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:24.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:24.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:24.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:24:24.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:24.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:24.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:24.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:24.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:24.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:24.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:24.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:24.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:24.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:24.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606], [object_number=93, tag_tid=29, entry_tid=5605], [object_number=92, tag_tid=29, entry_tid=5604]]], state=connected] 2026-03-24T02:24:24.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:24.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=95, tag_tid=29, entry_tid=5607' 2026-03-24T02:24:24.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=93, tag_tid=30, entry_tid=1' -a 'object_number=93, tag_tid=30, entry_tid=1' = 'object_number=95, tag_tid=29, entry_tid=5607' 2026-03-24T02:24:24.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=95, tag_tid=29, entry_tid=5607' '!=' '' 2026-03-24T02:24:24.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:24:24.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=95, tag_tid=29, entry_tid=5607' = '' 2026-03-24T02:24:24.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=95, tag_tid=29, entry_tid=5607' 2026-03-24T02:24:24.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:24.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:24:24.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:24:24.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=30 2026-03-24T02:24:24.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=95, tag_tid=29, entry_tid=5607' 2026-03-24T02:24:24.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:24:24.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:24:24.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=29 2026-03-24T02:24:24.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:24.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:24:24.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:24:24.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:24:24.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=95, tag_tid=29, entry_tid=5607' 2026-03-24T02:24:24.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:24:24.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:24:24.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=5607 2026-03-24T02:24:24.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 30 = 29 -a 1 -le 5607 2026-03-24T02:24:24.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:24:24.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:24:24.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:24:24.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:24:24.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:24.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:24.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:24.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:24:24.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:24:24.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:24:24.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:24:24.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:24:24.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:24.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:24.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:24:24.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:24:24.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:24:24.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:24:24.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:24:24.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:24:24.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:24:24.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:24:24.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:24:24.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:24.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:24:24.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:24:24.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:24:24.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:24:24.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:24:24.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:24:24.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:24:24.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:24.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:24:24.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:24.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:24:24.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:24.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:24:24.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:24:24.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:24:24.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:24:24.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:24:24.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:24:24.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:24:24.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:24:24.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:24.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:24.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:24.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:24:24.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:24:24.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:24:24.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:24:24.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:24:24.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:24.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:24.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:24.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:24:24.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:24.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:24:24.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:24.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:24.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:24.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:24:24.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:24.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:24.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:24.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:24:24.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:24.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:24.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:24.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:24.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:24.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:24.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:24.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:24.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:24.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:24.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606], [object_number=93, tag_tid=29, entry_tid=5605], [object_number=92, tag_tid=29, entry_tid=5604]]], state=connected] 2026-03-24T02:24:24.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:24.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:24.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:24:24.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:24.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:24.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:24.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:24:24.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:24.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:24.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:24.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:24:24.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:24.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:24.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:24.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:24.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:24.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:24.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:24.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:24.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:24.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:24.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606], [object_number=93, tag_tid=29, entry_tid=5605], [object_number=92, tag_tid=29, entry_tid=5604]]], state=connected] 2026-03-24T02:24:24.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:24.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=95, tag_tid=29, entry_tid=5607' 2026-03-24T02:24:24.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=93, tag_tid=30, entry_tid=1' -a 'object_number=93, tag_tid=30, entry_tid=1' = 'object_number=95, tag_tid=29, entry_tid=5607' 2026-03-24T02:24:24.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=95, tag_tid=29, entry_tid=5607' '!=' 'object_number=95, tag_tid=29, entry_tid=5607' 2026-03-24T02:24:24.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:24:24.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:24:24.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:24:24.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:24.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:24.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:24.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:24:24.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:24:24.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:24:24.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:24:24.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:24:24.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:24.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:24.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:24:24.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:24:24.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:24:24.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:24:24.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:24:24.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:24:24.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:24:24.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:24:24.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:24:24.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:24.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:24:24.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:24:24.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:24:24.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:24:24.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:24:24.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:24:24.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:24:24.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:24.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:24:24.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:24.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:24:24.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:24.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:24:24.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:24:24.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:24:24.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:24:24.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:24:24.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:24:24.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:24:24.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:24:24.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:24.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:24.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:24.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:24:24.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:24:24.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:24:24.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:24:24.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:24:24.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:24.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:24.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:24.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:24:25.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:25.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:24:25.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:25.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:25.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:25.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:24:25.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:25.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:25.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:25.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:24:25.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:25.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:25.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:25.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:25.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:25.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:25.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:25.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:25.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:25.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:25.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606], [object_number=93, tag_tid=29, entry_tid=5605], [object_number=92, tag_tid=29, entry_tid=5604]]], state=connected] 2026-03-24T02:24:25.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:25.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:25.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:24:25.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:25.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:25.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:25.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:24:25.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:25.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:25.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:25.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:24:25.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:25.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:25.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:25.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:25.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:25.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:25.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:25.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:25.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:25.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:25.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606], [object_number=93, tag_tid=29, entry_tid=5605], [object_number=92, tag_tid=29, entry_tid=5604]]], state=connected] 2026-03-24T02:24:25.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:25.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=95, tag_tid=29, entry_tid=5607' 2026-03-24T02:24:25.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=93, tag_tid=30, entry_tid=1' -a 'object_number=93, tag_tid=30, entry_tid=1' = 'object_number=95, tag_tid=29, entry_tid=5607' 2026-03-24T02:24:25.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=95, tag_tid=29, entry_tid=5607' '!=' 'object_number=95, tag_tid=29, entry_tid=5607' 2026-03-24T02:24:25.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:24:25.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:24:25.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:24:25.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:25.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:25.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:25.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:24:25.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:24:25.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:24:25.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:24:25.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:24:25.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:25.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:25.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:24:25.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:24:25.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:24:25.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:24:25.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:24:25.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:24:25.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:24:25.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:24:25.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:24:25.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:25.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:24:25.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:24:25.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:24:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:24:25.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:25.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:25.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:25.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:24:26.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:26.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:24:26.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:26.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:26.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:26.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:24:26.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:26.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:26.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:26.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:24:26.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:26.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:26.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:26.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:26.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:26.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:26.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:26.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:26.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:26.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:26.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:26.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:24:26.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:26.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:26.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:26.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:24:26.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:26.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:26.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:26.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:24:26.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:26.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:26.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:26.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:26.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:26.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:26.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:26.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:26.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:26.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:26.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:26.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:26.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:26.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=93, tag_tid=30, entry_tid=1' -a 'object_number=93, tag_tid=30, entry_tid=1' = 'object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:26.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:26.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror test snap10 2026-03-24T02:24:26.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:26.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:26.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:26.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap10 2026-03-24T02:24:26.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:24:26.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:24:26.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:24:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/test@snap10 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'test': 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 19 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c9689d5e36 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c9689d5e36 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:15:23 2026 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:23:41 2026 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:24:00 2026 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c9689d5e36 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:24:27.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:24:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n snap9-clean ']' 2026-03-24T02:24:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap9-clean 2026-03-24T02:24:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap9-clean 2026-03-24T02:24:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:24:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:27.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:27.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:27.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:28.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:28.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:28.733 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:28.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:28.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap10 2026-03-24T02:24:28.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:28.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:28.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap10 2026-03-24T02:24:28.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:24:28.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:28.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:28.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:28.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:29.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:29.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:30.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:30.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:30.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:30.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:30.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ clean_snap_name=snap10-clean 2026-03-24T02:24:30.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror test snap10-clean 2026-03-24T02:24:30.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:24:30.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:30.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:30.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap10-clean 2026-03-24T02:24:30.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10-clean --debug-rbd=20 --debug-journaler=20 2026-03-24T02:24:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-24T02:24:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-24T02:24:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:31.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:31.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:31.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:24:31.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:24:31.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:24:31.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:24:32.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-24T02:24:32.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:32.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:33.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:33.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:24:33.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:24:33.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:24:33.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-24T02:24:33.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:24:33.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:24:33.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:24:33.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:24:33.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:24:33.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:24:33.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:24:33.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:24:33.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:24:33.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:24:33.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:24:33.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-24T02:24:33.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:24:33.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:24:33.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:24:33.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:24:33.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:24:33.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:24:33.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:24:33.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:33.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:33.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:33.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:24:33.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:24:33.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:24:33.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:33.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:33.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:33.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-24T02:24:33.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:24:33.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:24:33.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:24:33.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:24:33.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:24:33.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:24:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:24:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:33.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:33.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:33.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:24:33.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:24:33.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:24:33.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:24:33.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:24:33.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:33.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:33.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:24:33.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:24:33.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:24:33.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:24:33.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:24:33.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:24:33.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:24:33.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:24:33.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:24:33.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:33.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:24:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:33.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:33.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:33.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:24:33.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:24:33.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:24:33.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:24:33.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:24:33.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:33.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:33.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:33.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:24:33.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:33.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:24:33.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:33.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:33.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:33.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:24:33.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:33.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:33.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:33.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:24:33.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:33.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:33.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:33.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:33.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:33.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:33.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:33.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:33.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:33.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=31, entry_tid=1], [object_number=92, tag_tid=31, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:33.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:33.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:33.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=93, tag_tid=31, entry_tid=1' 2026-03-24T02:24:33.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:24:33.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:33.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:33.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:33.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:24:33.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:33.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:33.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:33.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:24:33.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:33.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:33.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:33.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:33.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:33.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:33.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:33.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:33.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:33.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=31, entry_tid=1], [object_number=92, tag_tid=31, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:33.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:33.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:33.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:33.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=93, tag_tid=31, entry_tid=1' -a 'object_number=93, tag_tid=31, entry_tid=1' = 'object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:33.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=93, tag_tid=30, entry_tid=1' '!=' '' 2026-03-24T02:24:33.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:24:33.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=93, tag_tid=30, entry_tid=1' = '' 2026-03-24T02:24:33.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:33.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=93, tag_tid=31, entry_tid=1' 2026-03-24T02:24:33.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:24:33.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:24:33.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=31 2026-03-24T02:24:33.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:33.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:24:33.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:24:33.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=30 2026-03-24T02:24:33.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=93, tag_tid=31, entry_tid=1' 2026-03-24T02:24:33.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:24:33.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:24:33.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:24:33.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:33.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:24:33.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:24:33.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=1 2026-03-24T02:24:33.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 31 = 30 -a 1 -le 1 2026-03-24T02:24:33.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:24:33.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:24:33.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:24:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:24:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:33.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:24:33.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:24:33.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:24:33.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:24:33.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:24:33.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:33.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:33.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:24:33.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:24:33.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:24:33.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:24:33.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:24:33.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:24:33.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:24:33.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:33.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:24:33.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:24:33.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:24:33.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:24:33.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:24:33.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:33.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:33.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:33.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:24:33.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:33.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:24:33.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:33.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:33.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:33.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:24:33.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:33.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:33.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:33.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:24:33.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:33.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:33.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:33.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:33.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:33.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=31, entry_tid=1], [object_number=92, tag_tid=31, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:33.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:33.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=93, tag_tid=31, entry_tid=1' 2026-03-24T02:24:33.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:24:33.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:33.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:33.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:33.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:24:33.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:33.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:33.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:33.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:24:33.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:33.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:33.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:33.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:33.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:33.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:33.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:33.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:33.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:33.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=31, entry_tid=1], [object_number=92, tag_tid=31, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:33.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:33.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:33.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=93, tag_tid=31, entry_tid=1' -a 'object_number=93, tag_tid=31, entry_tid=1' = 'object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:33.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=93, tag_tid=30, entry_tid=1' '!=' 'object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:33.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:24:33.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:24:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:24:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:24:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:24:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:24:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:24:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:24:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:24:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:24:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:24:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:24:34.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:24:34.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:24:34.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:24:34.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:24:34.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:24:34.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:34.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:24:34.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:24:34.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:24:34.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:24:34.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:24:34.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:24:34.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:24:34.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:34.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:24:34.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:34.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:24:34.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:24:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:24:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:24:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:24:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:24:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:24:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:24:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:24:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:24:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:24:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:24:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:24:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:24:34.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:34.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:34.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:34.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:24:34.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:34.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:24:34.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:34.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:34.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:34.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:24:34.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:34.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:34.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:34.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:24:34.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:34.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:34.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:34.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:34.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:34.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:34.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:34.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:34.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:34.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=31, entry_tid=1], [object_number=92, tag_tid=31, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:34.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:34.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:34.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=93, tag_tid=31, entry_tid=1' 2026-03-24T02:24:34.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:24:34.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:34.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:34.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:34.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:24:34.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:34.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:34.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:34.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:24:34.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:34.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:34.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:34.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:34.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:34.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:34.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:34.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:34.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:34.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=31, entry_tid=1], [object_number=92, tag_tid=31, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:34.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:34.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:34.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:34.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=93, tag_tid=31, entry_tid=1' -a 'object_number=93, tag_tid=31, entry_tid=1' = 'object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:34.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=93, tag_tid=30, entry_tid=1' '!=' 'object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:34.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:24:34.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:24:35.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:24:35.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:35.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:35.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:35.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:24:35.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:24:35.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:24:35.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:24:35.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:24:35.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:35.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:35.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:24:35.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:24:35.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:24:35.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:24:35.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:24:35.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:24:35.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:24:35.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:24:35.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:24:35.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:35.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:24:35.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:24:35.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:24:35.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:24:35.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:24:35.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:24:35.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:24:35.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:35.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:24:35.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:35.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:24:35.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:35.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:24:35.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:24:35.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:24:35.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:24:35.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:24:35.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:24:35.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:24:35.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:24:35.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:35.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:35.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:35.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:24:35.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:24:35.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:24:35.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:24:35.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:24:35.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:35.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:35.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:35.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:24:35.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:35.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:24:35.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:35.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:35.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:35.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:24:35.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:35.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:35.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:35.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:24:35.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:35.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:35.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:35.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:35.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:35.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:35.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:35.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:35.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:35.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=31, entry_tid=1], [object_number=92, tag_tid=31, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:35.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:35.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:35.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=93, tag_tid=31, entry_tid=1' 2026-03-24T02:24:35.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:24:35.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:35.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:35.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:35.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:24:35.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:35.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:35.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:35.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:24:35.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:35.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:35.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:35.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:35.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:35.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:35.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:35.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:35.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:35.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=31, entry_tid=1], [object_number=92, tag_tid=31, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:35.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:35.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:35.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:35.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=93, tag_tid=31, entry_tid=1' -a 'object_number=93, tag_tid=31, entry_tid=1' = 'object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:35.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=93, tag_tid=30, entry_tid=1' '!=' 'object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:35.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:24:35.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:24:37.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:24:37.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:37.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:37.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:37.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:24:37.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:24:37.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:24:37.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:24:37.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:24:37.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:37.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:37.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:24:37.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:24:37.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:24:37.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:24:37.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:24:37.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:24:37.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:24:37.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:24:37.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:24:37.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:37.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:37.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:37.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:24:37.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:24:37.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:24:37.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:24:37.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:24:37.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:37.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:37.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:37.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:24:37.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:37.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:24:37.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:37.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:37.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:37.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:24:37.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:37.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:37.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:37.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:24:37.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:37.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:37.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:37.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:37.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:37.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:37.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:37.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:37.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:37.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=31, entry_tid=1], [object_number=92, tag_tid=31, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:37.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:37.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:37.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=93, tag_tid=31, entry_tid=1' 2026-03-24T02:24:37.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:24:37.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:37.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:37.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:37.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:24:37.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:37.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:37.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:37.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:24:37.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:37.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:37.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:37.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:37.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:37.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:37.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:37.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:37.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:37.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=31, entry_tid=1], [object_number=92, tag_tid=31, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:37.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:37.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:37.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:37.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=93, tag_tid=31, entry_tid=1' -a 'object_number=93, tag_tid=31, entry_tid=1' = 'object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:37.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=93, tag_tid=30, entry_tid=1' '!=' 'object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:37.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:24:37.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:24:39.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:24:39.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:39.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:39.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:39.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:24:39.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:24:39.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:24:39.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:24:39.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:24:39.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:39.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:39.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:24:39.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:24:39.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:24:39.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:24:39.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:24:39.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:24:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:24:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:24:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:24:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:24:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:24:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:24:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:24:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:24:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:24:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:24:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:24:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:24:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:39.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:24:39.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:24:39.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:24:39.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:24:39.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:24:39.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:24:39.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:24:39.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:24:39.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:39.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:39.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:39.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:24:39.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:24:39.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:24:39.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:24:39.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:24:39.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:39.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:39.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:39.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:24:39.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:39.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:24:39.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:39.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:39.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:39.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:24:39.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:39.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:39.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:39.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:24:39.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:39.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:39.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:39.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:39.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:39.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:39.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:39.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:39.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:39.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=31, entry_tid=1], [object_number=92, tag_tid=31, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:39.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:39.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:39.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=93, tag_tid=31, entry_tid=1' 2026-03-24T02:24:39.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:24:39.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:39.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:39.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:39.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:24:39.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:39.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:39.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:39.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:24:39.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:39.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:39.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:39.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:39.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:39.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:39.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:39.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:39.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:39.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=31, entry_tid=1], [object_number=92, tag_tid=31, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:39.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=93, tag_tid=30, entry_tid=1], [object_number=92, tag_tid=30, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:39.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:39.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:39.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=93, tag_tid=31, entry_tid=1' -a 'object_number=93, tag_tid=31, entry_tid=1' = 'object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:39.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=93, tag_tid=30, entry_tid=1' '!=' 'object_number=93, tag_tid=30, entry_tid=1' 2026-03-24T02:24:39.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:24:39.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:24:41.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror test 2026-03-24T02:24:41.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:41.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:41.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:41.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:24:41.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n test ']' 2026-03-24T02:24:41.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/test' 2026-03-24T02:24:41.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-03-24T02:24:41.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:24:41.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:41.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:41.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:24:41.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:24:41.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:24:41.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:24:41.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:24:41.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:24:41.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:24:41.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-03-24T02:24:41.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:24:41.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:24:41.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:24:41.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:24:41.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:24:41.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:24:41.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:24:41.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:24:41.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:24:41.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:41.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:24:41.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:41.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:24:41.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:41.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:24:41.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:24:41.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:24:41.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:24:41.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:24:41.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:24:41.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:24:41.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:24:41.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:24:41.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:24:41.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:24:41.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:24:41.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:24:41.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:24:41.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:24:41.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:24:41.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:41.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:41.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:24:41.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-03-24T02:24:41.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:41.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror test 2026-03-24T02:24:41.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:41.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:41.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:41.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '' 2026-03-24T02:24:41.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:41.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:41.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:41.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:24:41.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:41.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:41.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:41.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:41.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:41.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:41.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:41.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:41.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:41.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=31, entry_tid=1], [object_number=92, tag_tid=31, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:41.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=93, tag_tid=31, entry_tid=1], [object_number=92, tag_tid=31, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:41.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:41.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=93, tag_tid=31, entry_tid=1' 2026-03-24T02:24:41.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-03-24T02:24:41.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:41.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:41.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:41.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-03-24T02:24:41.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:24:41.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:24:41.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:24:41.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:24:41.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-test.status 2026-03-24T02:24:41.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-test.status 2026-03-24T02:24:41.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:24:41.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:41.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-03-24T02:24:41.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:41.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 23 2026-03-24T02:24:41.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 23 2026-03-24T02:24:41.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:24:41.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=31, entry_tid=1], [object_number=92, tag_tid=31, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:41.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=93, tag_tid=31, entry_tid=1], [object_number=92, tag_tid=31, entry_tid=0], [object_number=95, tag_tid=29, entry_tid=5607], [object_number=94, tag_tid=29, entry_tid=5606]]], state=connected] 2026-03-24T02:24:41.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-test.status 2026-03-24T02:24:41.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=93, tag_tid=31, entry_tid=1' 2026-03-24T02:24:41.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=93, tag_tid=31, entry_tid=1' -a 'object_number=93, tag_tid=31, entry_tid=1' = 'object_number=93, tag_tid=31, entry_tid=1' 2026-03-24T02:24:41.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:41.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror test snap10-clean 2026-03-24T02:24:41.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:24:41.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:41.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:41.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap10-clean 2026-03-24T02:24:41.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:24:41.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:24:41.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:24:42.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/test@snap10-clean 2026-03-24T02:24:42.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'test': 2026-03-24T02:24:42.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:24:42.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:24:42.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 20 2026-03-24T02:24:42.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c9689d5e36 2026-03-24T02:24:42.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c9689d5e36 2026-03-24T02:24:42.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:24:42.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:24:42.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:24:42.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:24:42.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:15:23 2026 2026-03-24T02:24:42.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:23:41 2026 2026-03-24T02:24:42.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:24:00 2026 2026-03-24T02:24:42.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:24:42.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c9689d5e36 2026-03-24T02:24:42.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:24:42.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:24:42.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 2d4a20c9-f6e9-4ce8-9d64-84febc265629 2026-03-24T02:24:42.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:24:42.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:42.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 1 10 2026-03-24T02:24:42.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:24:42.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap1 2026-03-24T02:24:42.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap1 2026-03-24T02:24:42.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:42.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:42.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap1 2026-03-24T02:24:42.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:24:42.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:42.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:42.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:42.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:43.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:43.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:44.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:44.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:44.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:44.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:44.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap1-clean 2026-03-24T02:24:44.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap1-clean 2026-03-24T02:24:44.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:44.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:44.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap1-clean 2026-03-24T02:24:44.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:24:44.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:44.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:44.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:44.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:45.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:45.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:45.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:45.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:45.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:46.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:46.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:24:46.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap2 2026-03-24T02:24:46.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap2 2026-03-24T02:24:46.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:46.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:46.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap2 2026-03-24T02:24:46.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:24:46.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:46.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:46.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:46.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:46.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:46.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:47.292 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:24:47.288+0000 7fc14cade640 -1 bdev(0x56417b647400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 6 2026-03-24T02:24:47.293 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:24:47.288+0000 7fc14c2dd640 -1 bdev(0x56417b647400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 6 2026-03-24T02:24:47.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:47.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:47.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:47.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:47.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap2-clean 2026-03-24T02:24:47.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap2-clean 2026-03-24T02:24:47.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:47.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:47.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap2-clean 2026-03-24T02:24:47.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:24:47.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:47.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:47.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:47.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:48.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:48.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:48.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:48.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:49.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:49.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:49.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:24:49.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap3 2026-03-24T02:24:49.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap3 2026-03-24T02:24:49.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:49.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:49.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap3 2026-03-24T02:24:49.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:24:49.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:49.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:49.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:49.280 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:49.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:49.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:50.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:50.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:51.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:51.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap3-clean 2026-03-24T02:24:51.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap3-clean 2026-03-24T02:24:51.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:51.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:51.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap3-clean 2026-03-24T02:24:51.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:24:51.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:51.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:51.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:51.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:51.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:51.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:52.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:52.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:52.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:52.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:52.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:24:52.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap4 2026-03-24T02:24:52.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap4 2026-03-24T02:24:52.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:52.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:52.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap4 2026-03-24T02:24:52.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:24:52.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:52.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:52.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:52.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:53.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:53.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:53.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:53.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:54.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:54.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:54.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap4-clean 2026-03-24T02:24:54.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap4-clean 2026-03-24T02:24:54.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:54.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:54.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap4-clean 2026-03-24T02:24:54.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:24:54.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:54.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:54.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:54.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:54.733 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:54.745 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:55.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:55.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:55.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:55.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:55.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:24:55.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap5 2026-03-24T02:24:55.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap5 2026-03-24T02:24:55.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:55.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:55.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap5 2026-03-24T02:24:55.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:24:55.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:55.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:55.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:55.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:56.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:56.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:57.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:57.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:57.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:57.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:57.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap5-clean 2026-03-24T02:24:57.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap5-clean 2026-03-24T02:24:57.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:57.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:57.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap5-clean 2026-03-24T02:24:57.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:24:57.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:57.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:57.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:57.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:58.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:58.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:59.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:24:59.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:59.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:24:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:24:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap6 2026-03-24T02:24:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap6 2026-03-24T02:24:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:24:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:24:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap6 2026-03-24T02:24:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:24:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:24:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:24:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:00.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:00.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:00.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:00.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:00.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:01.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:25:01.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap6-clean 2026-03-24T02:25:01.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap6-clean 2026-03-24T02:25:01.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:01.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:01.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap6-clean 2026-03-24T02:25:01.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:25:01.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:01.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:01.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:01.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:01.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:01.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:02.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:02.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:03.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:03.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:25:03.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:25:03.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap7 2026-03-24T02:25:03.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap7 2026-03-24T02:25:03.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:03.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:03.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap7 2026-03-24T02:25:03.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:25:03.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:03.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:03.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:03.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:03.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:03.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:04.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:04.614 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:04.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:04.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:25:04.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap7-clean 2026-03-24T02:25:04.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap7-clean 2026-03-24T02:25:04.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:04.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:04.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap7-clean 2026-03-24T02:25:04.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:25:04.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:04.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:04.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:04.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:05.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:05.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:06.139 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:25:06.136+0000 7fb26b2c5640 -1 bdev(0x55ac13011400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-03-24T02:25:06.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:06.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:06.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:06.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:25:06.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:25:06.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap8 2026-03-24T02:25:06.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap8 2026-03-24T02:25:06.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:06.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:06.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap8 2026-03-24T02:25:06.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:25:06.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:06.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:06.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:06.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:07.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:07.832 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:08.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:08.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:08.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:25:08.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap8-clean 2026-03-24T02:25:08.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap8-clean 2026-03-24T02:25:08.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:08.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:08.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap8-clean 2026-03-24T02:25:08.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:25:08.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:08.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:08.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:08.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:09.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:09.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:10.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap9 2026-03-24T02:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap9 2026-03-24T02:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap9 2026-03-24T02:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:11.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:11.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:12.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:12.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:12.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:12.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:25:12.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap9-clean 2026-03-24T02:25:12.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap9-clean 2026-03-24T02:25:12.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:12.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:12.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap9-clean 2026-03-24T02:25:12.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:25:12.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:12.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:12.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:12.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:13.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:13.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:13.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:13.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:14.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:14.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:25:14.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:25:14.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap10 2026-03-24T02:25:14.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap10 2026-03-24T02:25:14.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:14.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:14.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap10 2026-03-24T02:25:14.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:25:14.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:14.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:14.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:14.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:14.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:14.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:15.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:15.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:15.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:15.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:25:15.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap10-clean 2026-03-24T02:25:15.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror test snap10-clean 2026-03-24T02:25:15.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:15.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:15.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap10-clean 2026-03-24T02:25:15.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:25:15.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:15.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:15.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:15.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10-clean /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export 2026-03-24T02:25:16.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:16.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10-clean /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:17.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-24T02:25:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:17.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-test.export /tmp/tmp.l3GEA596QR/cluster1-mirror-test.export 2026-03-24T02:25:17.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:25:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 1 10 2026-03-24T02:25:17.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:25:17.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap1 2026-03-24T02:25:17.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap1 2026-03-24T02:25:17.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:17.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:17.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:17.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap1 2026-03-24T02:25:17.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1 2026-03-24T02:25:18.583 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-03-24T02:25:18.583 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-03-24T02:25:18.583 INFO:tasks.rbd_mirror_thrash:waiting for 7.278705536876084 secs before reviving daemons 2026-03-24T02:25:18.584 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:25:18.579+0000 7f5e0fa4d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 42582) UID: 1000 2026-03-24T02:25:18.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:18.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap1-clean 2026-03-24T02:25:18.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap1-clean 2026-03-24T02:25:18.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:18.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:18.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:18.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap1-clean 2026-03-24T02:25:18.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1-clean 2026-03-24T02:25:19.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:19.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:25:19.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap2 2026-03-24T02:25:19.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap2 2026-03-24T02:25:19.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:19.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:19.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:19.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap2 2026-03-24T02:25:19.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2 2026-03-24T02:25:20.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:20.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap2-clean 2026-03-24T02:25:20.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap2-clean 2026-03-24T02:25:20.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:20.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:20.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:20.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap2-clean 2026-03-24T02:25:20.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2-clean 2026-03-24T02:25:21.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:21.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:25:21.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap3 2026-03-24T02:25:21.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap3 2026-03-24T02:25:21.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:21.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:21.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:21.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap3 2026-03-24T02:25:21.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3 2026-03-24T02:25:22.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:22.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap3-clean 2026-03-24T02:25:22.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap3-clean 2026-03-24T02:25:22.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:22.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:22.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:22.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap3-clean 2026-03-24T02:25:22.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3-clean 2026-03-24T02:25:23.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:23.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:25:23.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap4 2026-03-24T02:25:23.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap4 2026-03-24T02:25:23.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:23.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:23.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:23.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap4 2026-03-24T02:25:23.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4 2026-03-24T02:25:24.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:24.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap4-clean 2026-03-24T02:25:24.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap4-clean 2026-03-24T02:25:24.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:24.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:24.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:24.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap4-clean 2026-03-24T02:25:24.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4-clean 2026-03-24T02:25:25.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:25.863 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-03-24T02:25:25.863 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:25:25.863 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-03-24T02:25:25.863 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-03-24T02:25:25.863 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-03-24T02:25:25.865 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-03-24T02:25:25.865 INFO:tasks.rbd_mirror_thrash:waiting for 102.46050074730593 secs before thrashing 2026-03-24T02:25:25.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:25:25.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap5 2026-03-24T02:25:25.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap5 2026-03-24T02:25:25.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:25.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:25.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:25.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap5 2026-03-24T02:25:25.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5 2026-03-24T02:25:26.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:26.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap5-clean 2026-03-24T02:25:26.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap5-clean 2026-03-24T02:25:26.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:26.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:26.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:26.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap5-clean 2026-03-24T02:25:26.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5-clean 2026-03-24T02:25:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:27.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:25:27.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap6 2026-03-24T02:25:27.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap6 2026-03-24T02:25:27.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:27.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:27.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:27.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap6 2026-03-24T02:25:27.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6 2026-03-24T02:25:28.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:28.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap6-clean 2026-03-24T02:25:28.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap6-clean 2026-03-24T02:25:28.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:28.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:28.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:28.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap6-clean 2026-03-24T02:25:28.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6-clean 2026-03-24T02:25:29.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:29.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:25:29.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap7 2026-03-24T02:25:29.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap7 2026-03-24T02:25:29.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:29.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:29.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:29.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap7 2026-03-24T02:25:29.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7 2026-03-24T02:25:30.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap7-clean 2026-03-24T02:25:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap7-clean 2026-03-24T02:25:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap7-clean 2026-03-24T02:25:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7-clean 2026-03-24T02:25:31.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:31.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:25:31.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap8 2026-03-24T02:25:31.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap8 2026-03-24T02:25:31.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:31.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:31.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:31.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap8 2026-03-24T02:25:31.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8 2026-03-24T02:25:32.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:32.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap8-clean 2026-03-24T02:25:32.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap8-clean 2026-03-24T02:25:32.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:32.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:32.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:32.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap8-clean 2026-03-24T02:25:32.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8-clean 2026-03-24T02:25:33.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:33.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:25:33.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap9 2026-03-24T02:25:33.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap9 2026-03-24T02:25:33.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:33.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:33.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:33.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap9 2026-03-24T02:25:33.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9 2026-03-24T02:25:34.519 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7366147503993146) > weight (0.25) 2026-03-24T02:25:34.519 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.6441180694931059) > weight (0.25) 2026-03-24T02:25:34.519 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.6645960825208713) > weight (0.25) 2026-03-24T02:25:34.519 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.9538783229513278) > weight (0.25) 2026-03-24T02:25:34.519 INFO:tasks.rbd_mirror_thrash:waiting for 71.20572809308472 secs before thrashing 2026-03-24T02:25:34.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap9-clean 2026-03-24T02:25:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap9-clean 2026-03-24T02:25:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap9-clean 2026-03-24T02:25:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9-clean 2026-03-24T02:25:35.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:35.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:25:35.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap10 2026-03-24T02:25:35.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap10 2026-03-24T02:25:35.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:35.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:35.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:35.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap10 2026-03-24T02:25:35.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10 2026-03-24T02:25:36.150 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:36.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap10-clean 2026-03-24T02:25:36.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror test snap10-clean 2026-03-24T02:25:36.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:36.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:36.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:36.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap10-clean 2026-03-24T02:25:36.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10-clean 2026-03-24T02:25:37.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:25:37.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror test 2026-03-24T02:25:37.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:37.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:37.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:37.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:25:37.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:25:37.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror test 2026-03-24T02:25:37.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:37.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:37.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:37.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-03-24T02:25:37.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-03-24T02:25:37.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-03-24T02:25:37.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:25:37.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-03-24T02:25:37.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:25:37.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:37.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:37.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:25:37.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:25:37.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:25:37.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:25:37.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror test 2026-03-24T02:25:37.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:25:37.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:25:37.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:25:37.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-24T02:25:37.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:25:37.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id=10c9689d5e36 2026-03-24T02:25:37.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:25:37.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:25:37.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror test deleted 10c9689d5e36 2026-03-24T02:25:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:25:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:25:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id=10c9689d5e36 2026-03-24T02:25:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:25:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:25:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror test 2026-03-24T02:25:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:25:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:25:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:25:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-24T02:25:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:25:37.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id=10c9689d5e36 2026-03-24T02:25:37.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 10c9689d5e36 2026-03-24T02:25:37.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -z 10c9689d5e36 -o 10c9689d5e36 = 10c9689d5e36 2026-03-24T02:25:37.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=present 2026-03-24T02:25:37.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = present 2026-03-24T02:25:37.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:25:37.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:25:38.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror test deleted 10c9689d5e36 2026-03-24T02:25:38.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:25:38.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:38.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=test 2026-03-24T02:25:38.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:25:38.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id=10c9689d5e36 2026-03-24T02:25:38.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:25:38.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:25:38.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror test 2026-03-24T02:25:38.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:25:38.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:25:38.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=test 2026-03-24T02:25:38.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-24T02:25:38.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:25:38.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-24T02:25:38.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:25:38.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:25:38.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:25:38.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:25:38.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ testlog 'TEST: create many images' 2026-03-24T02:25:38.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ tee -a /tmp/tmp.l3GEA596QR/rbd-mirror.test.log 2026-03-24T02:25:38.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ date '+%F %T' 2026-03-24T02:25:38.746 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo 2026-03-24 02:25:38 TEST: create many images 2026-03-24T02:25:38.746 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:2026-03-24 02:25:38 TEST: create many images 2026-03-24T02:25:38.747 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ snap_name=snap 2026-03-24T02:25:38.747 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 1 50 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_1 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_1 journal 128M 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_1 128M 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:25:38.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:25:38.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:25:38.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:38.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_1 2026-03-24T02:25:38.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:25:38.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:38.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:25:38.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:25:38.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:25:38.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:25:38.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_1 2026-03-24T02:25:38.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:38.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:38.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:25:38.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:25:38.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=14 2026-03-24T02:25:38.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:25:38.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:25:40.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 25634816 bytes 2026-03-24T02:25:41.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 50791424 bytes 2026-03-24T02:25:43.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 66797568 bytes 2026-03-24T02:25:46.114 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:25:46.107+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6240 finish: resending after timeout 2026-03-24T02:25:49.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:400 IOs, 136790016 bytes 2026-03-24T02:25:50.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:500 IOs, 177090560 bytes 2026-03-24T02:25:52.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:600 IOs, 207979520 bytes 2026-03-24T02:25:52.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:700 IOs, 208608256 bytes 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_2 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_2 journal 128M 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_2 128M 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:52.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:52.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:52.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:52.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:25:52.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:52.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:25:52.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:25:52.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:25:52.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:52.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_2 2026-03-24T02:25:52.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:25:52.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:52.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:25:52.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:25:52.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:25:52.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:25:52.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_2 2026-03-24T02:25:52.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:52.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:52.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:25:52.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:25:52.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=7 2026-03-24T02:25:52.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:25:52.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:25:53.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 7300096 bytes 2026-03-24T02:25:56.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 73811968 bytes 2026-03-24T02:25:59.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 93125632 bytes 2026-03-24T02:25:59.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:25:59.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:25:59.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:25:59.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:25:59.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_3 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_3 journal 128M 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_3 128M 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:25:59.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_3 2026-03-24T02:25:59.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:25:59.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:59.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:25:59.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:25:59.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:25:59.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:25:59.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_3 2026-03-24T02:25:59.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:25:59.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:25:59.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:25:59.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:25:59.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=5 2026-03-24T02:25:59.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:25:59.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:26:01.363 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:01.359+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a25140 finish: resending after timeout 2026-03-24T02:26:02.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 32121856 bytes 2026-03-24T02:26:04.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_4 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_4 journal 128M 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_4 128M 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:04.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_4 2026-03-24T02:26:05.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:26:05.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:05.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:26:05.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:26:05.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:26:05.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:26:05.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_4 2026-03-24T02:26:05.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:05.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:05.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:26:05.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:26:05.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=7 2026-03-24T02:26:05.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:26:05.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:26:06.364 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:06.359+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a25140 finish: resending after timeout 2026-03-24T02:26:07.369 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:07.363+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a25ec0 finish: resending after timeout 2026-03-24T02:26:09.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 44580864 bytes 2026-03-24T02:26:11.365 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:11.359+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a25140 finish: resending after timeout 2026-03-24T02:26:12.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:26:12.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:26:12.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:26:12.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:26:12.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:26:12.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_5 2026-03-24T02:26:12.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_5 journal 128M 2026-03-24T02:26:12.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:12.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:12.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:12.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:12.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:26:12.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:26:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:26:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_5 128M 2026-03-24T02:26:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:26:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:26:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:26:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:26:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_5 2026-03-24T02:26:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:26:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:26:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:26:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:26:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:26:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_5 2026-03-24T02:26:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:26:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:26:12.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=7 2026-03-24T02:26:12.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:26:12.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:26:12.370 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:12.363+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a25ec0 finish: resending after timeout 2026-03-24T02:26:13.373 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:13.367+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c83c0 finish: resending after timeout 2026-03-24T02:26:16.366 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:16.363+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a25140 finish: resending after timeout 2026-03-24T02:26:17.371 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:17.363+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a25ec0 finish: resending after timeout 2026-03-24T02:26:18.376 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:18.371+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c83c0 finish: resending after timeout 2026-03-24T02:26:18.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 59027456 bytes 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_6 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_6 journal 128M 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_6 128M 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:19.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:26:19.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:19.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:26:19.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:26:19.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:26:19.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:19.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_6 2026-03-24T02:26:19.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:26:19.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:19.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:26:19.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:26:19.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:26:19.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:26:19.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_6 2026-03-24T02:26:19.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:19.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:19.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:26:19.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:26:19.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=9 2026-03-24T02:26:19.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:26:19.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:26:20.274 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:20.267+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:26:21.366 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:21.359+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a25140 finish: resending after timeout 2026-03-24T02:26:22.372 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:22.367+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a25ec0 finish: resending after timeout 2026-03-24T02:26:23.377 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:23.371+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c83c0 finish: resending after timeout 2026-03-24T02:26:25.275 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:25.267+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:26:25.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 115758080 bytes 2026-03-24T02:26:26.367 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:26.363+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a25140 finish: resending after timeout 2026-03-24T02:26:27.372 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:27.367+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a25ec0 finish: resending after timeout 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_7 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_7 journal 128M 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_7 128M 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_7 2026-03-24T02:26:28.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:26:28.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:28.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:26:28.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:26:28.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:26:28.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:26:28.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_7 2026-03-24T02:26:28.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:28.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:28.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:26:28.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:26:28.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=12 2026-03-24T02:26:28.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:26:28.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:26:28.379 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:28.375+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c83c0 finish: resending after timeout 2026-03-24T02:26:30.275 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:30.271+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:26:31.368 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:31.363+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a25140 finish: resending after timeout 2026-03-24T02:26:32.373 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:32.367+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a25ec0 finish: resending after timeout 2026-03-24T02:26:32.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 39607296 bytes 2026-03-24T02:26:33.381 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:33.375+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c83c0 finish: resending after timeout 2026-03-24T02:26:34.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 59522048 bytes 2026-03-24T02:26:35.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 64546816 bytes 2026-03-24T02:26:35.277 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:35.271+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:26:37.373 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:37.367+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a25ec0 finish: resending after timeout 2026-03-24T02:26:38.384 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:38.379+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c83c0 finish: resending after timeout 2026-03-24T02:26:38.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:400 IOs, 124536832 bytes 2026-03-24T02:26:40.278 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:40.271+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_8 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_8 journal 128M 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_8 128M 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:26:40.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:26:40.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:26:40.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:40.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_8 2026-03-24T02:26:40.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:26:40.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:40.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:26:40.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:26:40.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:26:40.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:26:40.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_8 2026-03-24T02:26:40.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:40.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:40.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:26:40.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:26:40.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=7 2026-03-24T02:26:40.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:26:40.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:26:45.278 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:45.271+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:26:45.726 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6090120384708071) > weight (0.25) 2026-03-24T02:26:45.726 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9836118431480438) > weight (0.25) 2026-03-24T02:26:45.726 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.973364509739791) > weight (0.25) 2026-03-24T02:26:45.726 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.3170228307809728) > weight (0.25) 2026-03-24T02:26:45.726 INFO:tasks.rbd_mirror_thrash:waiting for 69.76575744709046 secs before thrashing 2026-03-24T02:26:46.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 57253888 bytes 2026-03-24T02:26:46.945 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:46.939+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c83c0 finish: resending after timeout 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_9 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_9 journal 128M 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_9 128M 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:47.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:26:47.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:47.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:26:47.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:26:47.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:26:47.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:26:47.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_9 2026-03-24T02:26:47.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:26:47.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:47.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:26:47.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:26:47.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:26:47.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:26:47.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_9 2026-03-24T02:26:47.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:26:47.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:26:47.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:26:47.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:26:47.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=14 2026-03-24T02:26:47.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:26:47.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:26:49.254 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:49.247+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:26:50.280 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:50.275+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:26:51.947 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:51.943+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c83c0 finish: resending after timeout 2026-03-24T02:26:52.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 50758656 bytes 2026-03-24T02:26:53.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 51332096 bytes 2026-03-24T02:26:54.260 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:54.255+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:26:55.282 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:55.275+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:26:59.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 148162560 bytes 2026-03-24T02:26:59.952 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:59.947+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c83c0 finish: resending after timeout 2026-03-24T02:26:59.955 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:26:59.951+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:27:01.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_10 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_10 journal 128M 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_10 128M 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:01.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_10 2026-03-24T02:27:01.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:27:01.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:01.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:27:01.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:27:01.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:27:01.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:27:01.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_10 2026-03-24T02:27:01.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:01.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:01.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:27:01.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:27:01.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=7 2026-03-24T02:27:01.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:27:01.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:27:02.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 623616 bytes 2026-03-24T02:27:02.739 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:02.735+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:27:06.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 48158720 bytes 2026-03-24T02:27:06.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 54450176 bytes 2026-03-24T02:27:08.326 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-03-24T02:27:08.326 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-03-24T02:27:08.326 INFO:tasks.rbd_mirror_thrash:waiting for 9.535330182493933 secs before reviving daemons 2026-03-24T02:27:08.327 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:27:08.323+0000 7f5507318640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 44775) UID: 1000 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_11 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_11 journal 128M 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_11 128M 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:27:08.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:27:08.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:08.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_11 2026-03-24T02:27:08.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:27:08.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:08.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:27:08.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:27:08.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:27:08.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:27:08.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_11 2026-03-24T02:27:08.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:08.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:08.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:27:08.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:27:08.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=7 2026-03-24T02:27:08.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:27:08.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_11 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:27:10.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:10.551+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:27:14.583 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:14.579+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c83c0 finish: resending after timeout 2026-03-24T02:27:14.583 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:14.579+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:27:14.583 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:14.579+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:27:15.306 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:15.300+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f8750 2026-03-24T02:27:15.307 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:15.304+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f9110 2026-03-24T02:27:15.307 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:15.304+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f91e0 2026-03-24T02:27:15.307 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:15.304+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f92b0 2026-03-24T02:27:15.307 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:15.304+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f9d40 2026-03-24T02:27:15.307 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:15.304+0000 7f5496ce4640 -1 reset not still connected to 0x5652c83cb930 2026-03-24T02:27:15.307 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:15.304+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c5dad0 2026-03-24T02:27:15.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:27:15.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:27:15.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:27:15.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:27:15.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:27:15.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_12 2026-03-24T02:27:15.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_12 journal 128M 2026-03-24T02:27:15.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:15.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:15.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:15.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:15.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:27:15.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:15.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:27:15.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:27:15.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:15.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_12 128M 2026-03-24T02:27:15.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:15.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:15.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:15.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:15.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:27:15.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:15.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:27:15.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:27:15.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:27:15.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:15.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_12 2026-03-24T02:27:15.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:27:15.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:15.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:27:15.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:27:15.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:27:15.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:27:15.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_12 2026-03-24T02:27:15.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:15.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:15.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:27:15.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:27:15.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=5 2026-03-24T02:27:15.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:27:15.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_12 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:27:16.501 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:16.496+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f8750 2026-03-24T02:27:16.501 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:16.496+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f9110 2026-03-24T02:27:16.501 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:16.496+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f91e0 2026-03-24T02:27:16.501 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:16.496+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f92b0 2026-03-24T02:27:16.501 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:16.496+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f9d40 2026-03-24T02:27:16.503 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:16.500+0000 7f5496ce4640 -1 reset not still connected to 0x5652c83cb930 2026-03-24T02:27:16.503 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:16.500+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c5dad0 2026-03-24T02:27:17.862 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-03-24T02:27:17.863 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:27:17.863 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-03-24T02:27:17.864 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-03-24T02:27:17.864 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-03-24T02:27:17.866 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-03-24T02:27:17.866 INFO:tasks.rbd_mirror_thrash:waiting for 99.19720660098429 secs before thrashing 2026-03-24T02:27:18.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 35817472 bytes 2026-03-24T02:27:19.584 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:19.579+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:27:19.584 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:19.579+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c83c0 finish: resending after timeout 2026-03-24T02:27:19.584 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:19.579+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_13 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_13 journal 128M 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_13 128M 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:27:20.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:27:20.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:27:20.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:20.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_13 2026-03-24T02:27:21.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:27:21.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:21.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:27:21.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:27:21.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:27:21.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:27:21.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_13 2026-03-24T02:27:21.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:21.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:21.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:27:21.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:27:21.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=7 2026-03-24T02:27:21.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:27:21.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_13 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:27:21.499 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:21.491+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:27:21.499 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:21.495+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:27:24.036 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:27:24.031+0000 7f0559634640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5564695cf2c0 finish: resending after timeout 2026-03-24T02:27:24.584 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:24.579+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:27:24.585 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:24.579+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c83c0 finish: resending after timeout 2026-03-24T02:27:24.585 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:24.579+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:27:25.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 46396416 bytes 2026-03-24T02:27:26.691 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:26.687+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_14 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_14 journal 128M 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_14 128M 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_14 2026-03-24T02:27:28.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:27:28.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:28.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:27:28.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:27:28.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:27:28.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:27:28.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_14 2026-03-24T02:27:28.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:28.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:28.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:27:28.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:27:28.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=11 2026-03-24T02:27:28.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:27:28.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_14 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:27:29.937 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:29.931+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:27:29.937 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:29.931+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:27:30.269 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:30.264+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f9110 2026-03-24T02:27:30.269 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:30.264+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f91e0 2026-03-24T02:27:30.273 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:30.268+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f92b0 2026-03-24T02:27:30.273 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:30.268+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f9d40 2026-03-24T02:27:30.273 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:30.268+0000 7f5496ce4640 -1 reset not still connected to 0x5652c83cb930 2026-03-24T02:27:30.274 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:27:30.272+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c5dad0 2026-03-24T02:27:31.009 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:31.003+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:27:31.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 59916288 bytes 2026-03-24T02:27:33.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 65274880 bytes 2026-03-24T02:27:33.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 77775872 bytes 2026-03-24T02:27:34.938 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:34.931+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:27:34.939 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:34.935+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:27:35.376 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:27:35.371+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b140 finish: resending after timeout 2026-03-24T02:27:35.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:400 IOs, 96548864 bytes 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_15 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_15 journal 128M 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_15 128M 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:39.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_15 2026-03-24T02:27:39.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:27:39.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:39.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:27:39.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:27:39.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:27:39.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:27:39.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_15 2026-03-24T02:27:39.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:39.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:39.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:27:39.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:27:39.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=12 2026-03-24T02:27:39.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:27:39.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_15 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:27:39.495 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:39.491+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:27:39.495 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:39.491+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:27:40.377 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:27:40.371+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b140 finish: resending after timeout 2026-03-24T02:27:42.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 43440128 bytes 2026-03-24T02:27:44.496 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:44.491+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:27:44.496 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:44.491+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:27:45.377 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:27:45.371+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b140 finish: resending after timeout 2026-03-24T02:27:46.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 103560192 bytes 2026-03-24T02:27:48.052 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:27:48.047+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:27:49.280 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 132276224 bytes 2026-03-24T02:27:50.379 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:27:50.375+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b140 finish: resending after timeout 2026-03-24T02:27:51.289 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:51.283+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:27:51.289 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:51.283+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:27:51.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:27:51.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:27:51.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:27:51.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:27:51.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:27:51.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_16 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_16 journal 128M 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_16 128M 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:51.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_16 2026-03-24T02:27:51.439 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:51.431+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:27:51.439 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:51.431+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:27:51.439 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:51.431+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:27:51.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:27:51.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:51.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:27:51.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:27:51.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:27:51.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:27:51.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_16 2026-03-24T02:27:51.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:51.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:51.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:27:51.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:27:51.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=8 2026-03-24T02:27:51.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:27:51.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_16 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:27:53.052 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:27:53.047+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:27:55.382 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:27:55.375+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b140 finish: resending after timeout 2026-03-24T02:27:55.492 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9873635227636535) > weight (0.25) 2026-03-24T02:27:55.492 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-03-24T02:27:55.493 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-03-24T02:27:55.493 INFO:tasks.rbd_mirror_thrash:waiting for 0.3597160922355591 secs before reviving daemons 2026-03-24T02:27:55.493 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:27:55.487+0000 7f0559e35640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 42487) UID: 1000 2026-03-24T02:27:55.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 46589952 bytes 2026-03-24T02:27:55.853 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-03-24T02:27:55.853 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:27:56.455 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:56.443+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:27:56.455 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:56.443+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:27:56.455 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:27:56.443+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:27:58.052 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:27:58.047+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_17 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_17 journal 128M 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_17 128M 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:27:59.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_17 2026-03-24T02:27:59.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:27:59.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:59.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:27:59.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:27:59.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:27:59.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:27:59.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_17 2026-03-24T02:27:59.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:27:59.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:27:59.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:27:59.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:27:59.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=10 2026-03-24T02:27:59.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:27:59.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_17 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:28:00.383 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:28:00.375+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b140 finish: resending after timeout 2026-03-24T02:28:01.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 15072256 bytes 2026-03-24T02:28:01.939 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:01.935+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:28:01.939 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:01.935+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:28:02.500 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:02.495+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:28:02.501 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:02.495+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:28:02.501 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:02.495+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:28:03.053 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:28:03.047+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:28:05.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 59939840 bytes 2026-03-24T02:28:06.940 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:06.935+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:28:07.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 86069248 bytes 2026-03-24T02:28:07.393 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:07.387+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8180 finish: resending after timeout 2026-03-24T02:28:07.501 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:07.495+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:28:07.501 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:07.495+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:28:07.501 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:07.495+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:28:08.053 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:28:08.047+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:28:09.023 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:09.019+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_18 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_18 journal 128M 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_18 128M 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:09.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_18 2026-03-24T02:28:10.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:28:10.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:10.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:28:10.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:28:10.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:28:10.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:28:10.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_18 2026-03-24T02:28:10.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:10.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:10.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:28:10.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:28:10.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=5 2026-03-24T02:28:10.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:28:10.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_18 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:28:11.940 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:11.935+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:28:12.393 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:12.387+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8180 finish: resending after timeout 2026-03-24T02:28:12.502 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:12.495+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:28:12.502 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:12.495+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:28:12.502 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:12.495+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:28:14.024 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:14.019+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:28:14.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 58070016 bytes 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_19 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_19 journal 128M 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_19 128M 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:28:15.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:28:15.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:28:15.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:15.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_19 2026-03-24T02:28:15.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:28:15.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:15.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:28:15.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:28:15.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:28:15.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:28:15.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_19 2026-03-24T02:28:15.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:15.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:15.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:28:15.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:28:15.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=8 2026-03-24T02:28:15.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:28:15.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_19 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:28:16.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 614400 bytes 2026-03-24T02:28:16.942 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:16.935+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:28:17.394 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:17.387+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8180 finish: resending after timeout 2026-03-24T02:28:17.503 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:17.495+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:28:17.503 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:17.499+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:28:18.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 25550848 bytes 2026-03-24T02:28:19.025 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:19.019+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:28:19.750 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:19.743+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:28:21.607 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:28:21.603+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b380 finish: resending after timeout 2026-03-24T02:28:21.948 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:21.943+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:28:22.395 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:22.387+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8180 finish: resending after timeout 2026-03-24T02:28:22.503 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:22.499+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:28:22.503 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:22.499+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:28:23.095 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:28:23.091+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5440 finish: resending after timeout 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_20 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_20 journal 128M 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_20 128M 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:28:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:28:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:28:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:28:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_20 2026-03-24T02:28:23.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:28:23.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:23.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:28:23.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:28:23.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:28:23.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:28:23.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_20 2026-03-24T02:28:23.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:23.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:23.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:28:23.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:28:23.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=10 2026-03-24T02:28:23.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:28:23.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_20 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:28:24.025 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:24.019+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:28:24.751 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:24.747+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:28:26.608 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:28:26.603+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b380 finish: resending after timeout 2026-03-24T02:28:26.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 37824512 bytes 2026-03-24T02:28:26.948 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:26.943+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:28:27.396 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:27.391+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8180 finish: resending after timeout 2026-03-24T02:28:27.504 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:27.499+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:28:27.505 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:27.499+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:28:27.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 43367424 bytes 2026-03-24T02:28:28.096 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:28:28.091+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5440 finish: resending after timeout 2026-03-24T02:28:29.027 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:29.019+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:28:29.752 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:29.747+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:28:30.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 96473088 bytes 2026-03-24T02:28:31.608 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:28:31.603+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b380 finish: resending after timeout 2026-03-24T02:28:31.949 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:31.943+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:28:32.396 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:32.391+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8180 finish: resending after timeout 2026-03-24T02:28:32.505 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:32.499+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:28:32.505 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:32.499+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:28:33.097 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:28:33.091+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5440 finish: resending after timeout 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_21 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_21 journal 128M 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_21 128M 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_21 2026-03-24T02:28:33.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:28:33.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:33.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:28:33.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:28:33.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:28:33.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:28:33.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_21 2026-03-24T02:28:33.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:33.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:33.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:28:33.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:28:33.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=11 2026-03-24T02:28:33.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:28:33.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_21 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:28:34.027 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:34.019+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:28:34.752 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:34.747+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:28:36.609 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:28:36.603+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b380 finish: resending after timeout 2026-03-24T02:28:37.506 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:37.499+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:28:37.506 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:37.499+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:28:38.099 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:28:38.095+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5440 finish: resending after timeout 2026-03-24T02:28:38.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 54583296 bytes 2026-03-24T02:28:39.027 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:39.023+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:28:40.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 89210880 bytes 2026-03-24T02:28:41.610 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:28:41.603+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b380 finish: resending after timeout 2026-03-24T02:28:41.959 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:41.951+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:28:41.959 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:41.951+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8180 finish: resending after timeout 2026-03-24T02:28:41.959 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:41.955+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:28:41.959 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:41.955+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:28:42.507 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:42.499+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:28:42.507 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:42.499+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:28:42.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 117137408 bytes 2026-03-24T02:28:43.101 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:28:43.095+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5440 finish: resending after timeout 2026-03-24T02:28:44.028 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:44.023+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:28:44.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:28:44.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:28:44.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:28:44.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:28:44.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:28:44.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_22 2026-03-24T02:28:44.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_22 journal 128M 2026-03-24T02:28:44.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:44.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:44.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:44.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:44.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:28:44.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:44.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:28:44.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:28:44.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:44.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_22 128M 2026-03-24T02:28:44.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:44.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:44.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:44.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:44.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:28:44.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:44.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:28:44.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:28:44.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:28:44.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:44.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_22 2026-03-24T02:28:44.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:28:44.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:44.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:28:44.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:28:44.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:28:44.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:28:44.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_22 2026-03-24T02:28:44.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:44.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:44.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:28:44.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:28:44.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=11 2026-03-24T02:28:44.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:28:44.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_22 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:28:46.610 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:28:46.603+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b380 finish: resending after timeout 2026-03-24T02:28:46.960 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:46.955+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:28:46.960 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:46.955+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:28:46.960 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:46.955+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8180 finish: resending after timeout 2026-03-24T02:28:46.960 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:46.955+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:28:47.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 29274112 bytes 2026-03-24T02:28:47.508 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:47.503+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:28:47.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 52734976 bytes 2026-03-24T02:28:48.101 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:28:48.095+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5440 finish: resending after timeout 2026-03-24T02:28:49.030 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:49.023+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:28:51.960 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:51.955+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:28:51.960 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:51.955+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:28:51.960 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:51.955+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:28:52.508 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:52.503+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:28:52.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 95110144 bytes 2026-03-24T02:28:54.031 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:54.023+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:28:55.243 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:28:55.239+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5440 finish: resending after timeout 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_23 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_23 journal 128M 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_23 128M 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:28:55.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_23 2026-03-24T02:28:56.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:28:56.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:56.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:28:56.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:28:56.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:28:56.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:28:56.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_23 2026-03-24T02:28:56.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:28:56.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:28:56.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:28:56.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:28:56.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=6 2026-03-24T02:28:56.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:28:56.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_23 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:28:56.965 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:56.959+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:28:56.966 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:56.959+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:28:56.966 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:56.959+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:28:56.967 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:56.959+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:28:57.064 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.288987790188191) > weight (0.25) 2026-03-24T02:28:57.064 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-03-24T02:28:57.064 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-03-24T02:28:57.064 INFO:tasks.rbd_mirror_thrash:waiting for 4.021533550272341 secs before reviving daemons 2026-03-24T02:28:57.065 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:28:57.059+0000 7f607d730640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 29661) UID: 1000 2026-03-24T02:28:57.456 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:28:57.451+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b380 finish: resending after timeout 2026-03-24T02:28:57.456 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:28:57.451+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b740 finish: resending after timeout 2026-03-24T02:28:57.509 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:57.503+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:28:58.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 22131712 bytes 2026-03-24T02:28:59.033 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:28:59.027+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:29:00.245 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:00.239+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5440 finish: resending after timeout 2026-03-24T02:29:01.087 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-03-24T02:29:01.087 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:29:01.087 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-03-24T02:29:01.087 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-03-24T02:29:01.087 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-03-24T02:29:01.089 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-03-24T02:29:01.089 INFO:tasks.rbd_mirror_thrash:waiting for 97.0478606853693 secs before thrashing 2026-03-24T02:29:01.975 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:01.971+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:29:01.975 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:01.971+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:29:01.976 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:01.971+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:29:01.976 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:01.971+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:29:02.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:29:02.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:29:02.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_24 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_24 journal 128M 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_24 128M 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:02.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_24 2026-03-24T02:29:02.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:29:02.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:02.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:29:02.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:29:02.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:29:02.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:29:02.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_24 2026-03-24T02:29:02.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:02.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:02.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:29:02.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:29:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=12 2026-03-24T02:29:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:29:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_24 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:29:02.509 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:02.503+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:29:03.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 8565760 bytes 2026-03-24T02:29:04.034 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:04.027+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:29:04.096 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:04.091+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:29:05.247 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:05.239+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5440 finish: resending after timeout 2026-03-24T02:29:05.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 27741184 bytes 2026-03-24T02:29:06.153 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:06.147+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b380 finish: resending after timeout 2026-03-24T02:29:06.153 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:06.147+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b740 finish: resending after timeout 2026-03-24T02:29:06.976 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:06.971+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:29:06.976 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:06.971+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:29:06.977 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:06.971+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:29:06.977 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:06.971+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:29:07.511 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:07.503+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:29:08.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 75958272 bytes 2026-03-24T02:29:09.034 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:09.027+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:29:09.097 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:09.091+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:29:10.248 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:10.243+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5440 finish: resending after timeout 2026-03-24T02:29:11.154 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:11.147+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b380 finish: resending after timeout 2026-03-24T02:29:11.154 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:11.147+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b740 finish: resending after timeout 2026-03-24T02:29:11.977 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:11.971+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:29:11.977 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:11.971+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:29:11.978 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:11.971+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:29:11.978 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:11.971+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:29:12.512 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:12.507+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:29:13.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:400 IOs, 134040576 bytes 2026-03-24T02:29:14.035 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:14.031+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:29:14.097 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:14.091+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_25 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_25 journal 128M 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_25 128M 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:14.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:29:14.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:14.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:29:14.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:29:14.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:29:14.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:14.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_25 2026-03-24T02:29:14.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:29:14.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:14.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:29:14.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:29:14.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:29:14.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:29:14.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_25 2026-03-24T02:29:14.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:14.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:14.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:29:14.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:29:14.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=5 2026-03-24T02:29:14.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:29:14.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_25 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:29:15.249 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:15.243+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5440 finish: resending after timeout 2026-03-24T02:29:16.155 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:16.147+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b380 finish: resending after timeout 2026-03-24T02:29:16.155 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:16.147+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b740 finish: resending after timeout 2026-03-24T02:29:16.977 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:16.971+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:29:16.978 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:16.971+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:29:16.979 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:16.975+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:29:16.979 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:16.975+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:29:17.512 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:17.507+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:29:18.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 48956416 bytes 2026-03-24T02:29:19.036 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:19.031+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:29:19.098 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:19.091+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:29:19.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:29:19.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:29:19.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:29:19.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:29:19.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:29:19.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_26 2026-03-24T02:29:19.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_26 journal 128M 2026-03-24T02:29:19.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:19.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:19.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:19.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:19.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:29:19.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:19.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:29:19.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:29:19.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:19.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_26 128M 2026-03-24T02:29:19.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:19.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:19.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:19.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:19.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:29:19.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:19.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:29:19.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:29:19.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:29:19.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:19.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_26 2026-03-24T02:29:19.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:29:19.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:19.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:29:19.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:29:19.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:29:19.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:29:19.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_26 2026-03-24T02:29:19.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:19.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:19.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:29:19.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:29:19.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=10 2026-03-24T02:29:19.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:29:19.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_26 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:29:20.252 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:20.247+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5440 finish: resending after timeout 2026-03-24T02:29:21.156 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:21.151+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b380 finish: resending after timeout 2026-03-24T02:29:21.156 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:21.151+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b740 finish: resending after timeout 2026-03-24T02:29:21.979 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:21.971+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:29:21.979 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:21.975+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:29:21.979 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:21.975+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:29:21.979 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:21.975+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:29:22.513 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:22.507+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:29:23.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 70349824 bytes 2026-03-24T02:29:23.391 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:23.383+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:29:24.037 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:24.031+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:29:24.099 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:24.095+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:29:25.252 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:25.247+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5440 finish: resending after timeout 2026-03-24T02:29:25.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 96824320 bytes 2026-03-24T02:29:26.156 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:26.151+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b380 finish: resending after timeout 2026-03-24T02:29:26.156 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:26.151+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b740 finish: resending after timeout 2026-03-24T02:29:26.963 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:26.959+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:29:26.980 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:26.975+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:29:26.980 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:26.975+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:29:26.980 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:26.975+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:29:26.980 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:26.975+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:29:27.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 126022656 bytes 2026-03-24T02:29:27.514 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:27.507+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:29:27.617 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:29:27.611+0000 7f085e5d7640 -1 reset not still connected to 0x55878595e5b0 2026-03-24T02:29:27.617 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:29:27.611+0000 7f085e5d7640 -1 reset not still connected to 0x55878d55fad0 2026-03-24T02:29:28.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:400 IOs, 133101568 bytes 2026-03-24T02:29:28.391 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:28.387+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:29:29.102 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:29.095+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_27 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_27 journal 128M 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_27 128M 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:29.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_27 2026-03-24T02:29:29.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:29:29.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:29.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:29:29.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:29:29.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:29:29.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:29:29.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_27 2026-03-24T02:29:29.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:29.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:29.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:29:29.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:29:29.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=12 2026-03-24T02:29:29.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:29:29.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_27 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:29:31.981 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:31.975+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:29:31.981 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:31.975+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:29:31.982 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:31.975+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:29:32.036 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:32.031+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:29:33.392 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:33.387+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:29:33.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 39299072 bytes 2026-03-24T02:29:34.104 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:34.099+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:29:35.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 71788544 bytes 2026-03-24T02:29:36.133 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:36.127+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b740 finish: resending after timeout 2026-03-24T02:29:36.133 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:36.127+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:29:36.493 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:29:36.485+0000 7fb2796c9640 -1 reset not still connected to 0x55ac15191040 2026-03-24T02:29:36.498 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:29:36.493+0000 7fb2796c9640 -1 reset not still connected to 0x55ac15254b60 2026-03-24T02:29:36.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 72398848 bytes 2026-03-24T02:29:36.981 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:36.975+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:29:36.982 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:36.975+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:29:36.984 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:36.979+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:29:37.038 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:37.031+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:29:37.168 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:37.163+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:29:38.392 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:38.387+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:29:39.107 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:39.099+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:29:41.133 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:41.127+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b740 finish: resending after timeout 2026-03-24T02:29:41.134 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:41.127+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_28 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_28 journal 128M 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_28 128M 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_28 2026-03-24T02:29:41.982 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:41.975+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:29:41.983 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:41.979+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:29:41.985 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:41.979+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:29:42.039 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:42.035+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:29:42.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:29:42.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:42.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:29:42.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:29:42.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:29:42.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:29:42.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_28 2026-03-24T02:29:42.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:42.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:42.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:29:42.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:29:42.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=9 2026-03-24T02:29:42.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:29:42.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_28 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:29:42.169 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:42.163+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:29:43.393 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:43.387+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:29:44.108 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:44.103+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:29:45.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 38328320 bytes 2026-03-24T02:29:46.134 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:46.127+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b740 finish: resending after timeout 2026-03-24T02:29:46.134 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:46.127+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:29:46.754 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:29:46.747+0000 7f085e5d7640 -1 reset not still connected to 0x55878388ca90 2026-03-24T02:29:46.754 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:29:46.747+0000 7f085e5d7640 -1 reset not still connected to 0x55878388d2b0 2026-03-24T02:29:47.169 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:47.163+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:29:48.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 64326656 bytes 2026-03-24T02:29:49.108 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:49.103+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:29:50.491 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:50.487+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:29:50.492 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:50.487+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:29:51.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_29 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_29 journal 128M 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_29 128M 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:51.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:51.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:51.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:29:51.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:51.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:29:51.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:29:51.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:29:51.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:29:51.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_29 2026-03-24T02:29:51.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:29:51.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:51.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:29:51.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:29:51.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:29:51.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:29:51.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_29 2026-03-24T02:29:51.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:29:51.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:29:51.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:29:51.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:29:51.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=14 2026-03-24T02:29:51.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:29:51.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_29 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:29:51.689 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:51.683+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:29:51.689 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:51.683+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:29:51.689 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:51.683+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:29:51.689 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:51.683+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:29:51.689 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:51.683+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:29:52.170 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:52.163+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:29:54.109 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:54.103+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:29:55.169 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:55.163+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b740 finish: resending after timeout 2026-03-24T02:29:55.169 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:29:55.163+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:29:55.493 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:55.487+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:29:55.493 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:55.487+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:29:56.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 36632576 bytes 2026-03-24T02:29:56.689 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:56.683+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:29:56.689 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:56.683+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:29:56.689 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:56.683+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:29:56.689 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:56.683+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:29:56.689 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:29:56.683+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:29:57.171 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:57.163+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:29:57.390 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 44279808 bytes 2026-03-24T02:29:59.109 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:29:59.103+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:29:59.531 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:29:59.527+0000 7f085e5d7640 -1 reset not still connected to 0x5587a614c340 2026-03-24T02:29:59.536 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:29:59.531+0000 7f085e5d7640 -1 reset not still connected to 0x5587a6ebe9c0 2026-03-24T02:30:00.494 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:00.487+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:30:00.494 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:00.487+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:30:01.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 107214848 bytes 2026-03-24T02:30:01.696 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:01.691+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:30:01.697 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:01.691+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:30:01.698 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:01.691+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:30:01.698 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:01.691+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:30:01.698 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:01.691+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:30:01.698 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:01.691+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:30:02.171 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:02.167+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:30:03.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:400 IOs, 135833600 bytes 2026-03-24T02:30:04.110 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:04.103+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:30:04.180 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:04.175+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b740 finish: resending after timeout 2026-03-24T02:30:04.181 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:04.175+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_30 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_30 journal 128M 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_30 128M 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_30 2026-03-24T02:30:05.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:30:05.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:05.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:30:05.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:30:05.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:30:05.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:30:05.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_30 2026-03-24T02:30:05.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:05.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:05.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:30:05.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:30:05.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=10 2026-03-24T02:30:05.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:30:05.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_30 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:30:05.494 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:05.487+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:30:05.495 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:05.487+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:30:06.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 4566016 bytes 2026-03-24T02:30:06.697 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:06.691+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:30:06.697 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:06.691+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:30:06.698 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:06.691+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:30:06.698 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:06.691+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:30:06.698 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:06.691+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:30:06.698 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:06.691+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:30:07.172 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:07.167+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:30:09.111 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:09.103+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:30:09.181 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:09.175+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b740 finish: resending after timeout 2026-03-24T02:30:09.181 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:09.175+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:30:09.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 54567936 bytes 2026-03-24T02:30:11.698 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:11.691+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:30:11.698 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:11.691+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:30:11.698 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:11.691+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:30:11.699 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:11.695+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:30:11.699 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:11.695+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:30:11.699 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:11.695+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:30:11.708 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:11.703+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:30:11.708 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:11.703+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:30:12.173 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:12.167+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:30:13.654 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:13.647+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:30:14.111 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:14.107+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:30:14.182 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:14.175+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b740 finish: resending after timeout 2026-03-24T02:30:14.182 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:14.175+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:30:15.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 139075584 bytes 2026-03-24T02:30:15.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:30:15.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:30:15.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:30:15.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:30:15.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_31 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_31 journal 128M 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_31 128M 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:15.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_31 2026-03-24T02:30:15.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:30:15.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:15.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:30:15.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:30:15.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:30:15.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:30:15.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_31 2026-03-24T02:30:15.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:15.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:15.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:30:15.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:30:15.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=7 2026-03-24T02:30:15.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:30:15.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_31 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:30:16.698 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:16.691+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:30:16.698 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:16.691+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:30:16.699 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:16.695+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:30:16.699 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:16.695+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:30:16.699 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:16.695+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:30:16.699 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:16.695+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:30:16.708 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:16.703+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:30:16.708 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:16.703+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:30:16.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 9230336 bytes 2026-03-24T02:30:17.173 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:17.167+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:30:18.654 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:18.647+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:30:19.112 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:19.107+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:30:19.183 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:19.175+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b740 finish: resending after timeout 2026-03-24T02:30:19.217 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:19.175+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:30:21.699 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:21.691+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:30:21.699 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:21.691+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:30:21.699 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:21.695+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2f09a40 finish: resending after timeout 2026-03-24T02:30:21.699 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:21.695+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:30:21.700 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:21.695+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:30:21.700 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:21.695+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:30:21.709 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:21.703+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:30:21.712 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:21.703+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:30:22.174 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:22.167+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:30:22.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 96471040 bytes 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_32 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_32 journal 128M 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_32 128M 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:22.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:30:22.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:30:22.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:30:22.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:22.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_32 2026-03-24T02:30:23.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:30:23.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:23.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:30:23.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:30:23.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:30:23.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:30:23.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_32 2026-03-24T02:30:23.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:23.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:23.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:30:23.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:30:23.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=8 2026-03-24T02:30:23.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:30:23.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_32 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:30:23.655 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:23.651+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:30:24.114 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:24.107+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:30:24.183 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:24.179+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:30:24.183 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:24.179+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b740 finish: resending after timeout 2026-03-24T02:30:25.029 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:25.023+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:30:26.699 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:26.695+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:30:26.700 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:26.695+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:30:26.700 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:26.695+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:30:26.700 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:26.695+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:30:26.700 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:26.695+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:30:26.709 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:26.703+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:30:26.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 44252160 bytes 2026-03-24T02:30:27.174 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:27.167+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:30:28.383 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-03-24T02:30:28.384 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-03-24T02:30:28.384 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-03-24T02:30:28.385 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-03-24T02:30:28.385 INFO:tasks.rbd_mirror_thrash:waiting for 89.94920719829133 secs before thrashing 2026-03-24T02:30:28.658 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:28.651+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:30:29.114 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:29.107+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:30:29.806 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:29.799+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:30:30.031 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:30.027+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:30:30.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 93379584 bytes 2026-03-24T02:30:31.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:30:31.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:30:31.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:30:31.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:30:31.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:30:31.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_33 2026-03-24T02:30:31.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_33 journal 128M 2026-03-24T02:30:31.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:31.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:31.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:31.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_33 2026-03-24T02:30:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:30:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:30:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_33 128M 2026-03-24T02:30:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_33 2026-03-24T02:30:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:30:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:30:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:30:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_33 2026-03-24T02:30:31.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:30:31.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:31.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:30:31.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:30:31.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:30:31.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:30:31.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_33 2026-03-24T02:30:31.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:31.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:31.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_33 2026-03-24T02:30:31.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:30:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=12 2026-03-24T02:30:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:30:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_33 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:30:31.701 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:31.695+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:30:31.701 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:31.695+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:30:31.702 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:31.695+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:30:31.702 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:31.695+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:30:31.702 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:31.695+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:30:31.710 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:31.703+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:30:32.175 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:32.171+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:30:33.877 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:33.871+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:30:33.878 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:33.871+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b740 finish: resending after timeout 2026-03-24T02:30:33.878 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:33.871+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:30:34.115 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:34.107+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:30:34.806 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:34.799+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:30:35.032 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:35.027+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:30:36.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 86009856 bytes 2026-03-24T02:30:36.703 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:36.695+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:30:36.703 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:36.695+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:30:36.703 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:36.695+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:30:36.703 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:36.695+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:30:36.711 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:36.703+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:30:38.138 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.6501940346134657) > weight (0.25) 2026-03-24T02:30:38.138 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.34203717893443264) > weight (0.25) 2026-03-24T02:30:38.138 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.416951278334998) > weight (0.25) 2026-03-24T02:30:38.138 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.2758997647559204) > weight (0.25) 2026-03-24T02:30:38.138 INFO:tasks.rbd_mirror_thrash:waiting for 115.08312464800713 secs before thrashing 2026-03-24T02:30:38.878 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:38.871+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:30:38.878 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:38.871+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:30:39.459 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:39.451+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:30:39.577 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:30:39.571+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:30:39.807 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:39.799+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:30:40.879 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:40.875+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:30:40.879 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:40.875+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f135a02480 finish: resending after timeout 2026-03-24T02:30:40.879 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:40.875+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:30:40.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 132308992 bytes 2026-03-24T02:30:41.704 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:41.699+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:30:41.704 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:41.699+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:30:41.704 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:41.699+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:30:41.704 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:41.699+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:30:41.711 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:41.707+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_34 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_34 journal 128M 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_34 128M 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:30:43.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:43.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:30:43.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:30:43.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:30:43.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:43.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_34 2026-03-24T02:30:43.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:30:43.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:43.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:30:43.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:30:43.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:30:43.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:30:43.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_34 2026-03-24T02:30:43.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:43.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:43.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:30:43.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:30:43.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=5 2026-03-24T02:30:43.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:30:43.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_34 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:30:43.879 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:43.871+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:30:43.879 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:43.875+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:30:44.578 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:30:44.571+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:30:45.899 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:45.891+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:30:46.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 40279040 bytes 2026-03-24T02:30:46.704 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:46.699+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:30:46.704 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:46.699+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:30:46.705 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:46.699+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:30:46.705 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:46.699+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:30:48.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_35 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_35 journal 128M 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_35 128M 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_35 2026-03-24T02:30:48.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:30:48.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:48.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:30:48.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:30:48.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:30:48.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:30:48.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_35 2026-03-24T02:30:48.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:48.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:48.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:30:48.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:30:48.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=10 2026-03-24T02:30:48.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:30:48.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_35 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:30:48.880 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:48.875+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:30:48.880 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:48.875+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:30:49.580 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:30:49.571+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:30:52.156 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:30:52.151+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:30:53.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 41178112 bytes 2026-03-24T02:30:53.881 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:53.875+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:30:53.882 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:53.875+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:30:54.582 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:30:54.575+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:30:54.614 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:54.607+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:30:54.615 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:54.607+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:30:54.615 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:54.611+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:30:54.617 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:54.611+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:30:54.617 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:54.611+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:30:55.360 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:55.355+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:30:55.360 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:30:55.355+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:30:55.683 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:30:55.675+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:30:56.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 70437888 bytes 2026-03-24T02:30:57.157 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:30:57.151+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_36 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_36 journal 128M 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_36 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_36 128M 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_36 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:30:58.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_36 2026-03-24T02:30:58.881 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:58.875+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:30:58.882 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:58.875+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:30:58.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:30:58.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:58.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:30:58.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:30:58.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:30:58.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:30:58.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_36 2026-03-24T02:30:58.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:30:58.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:30:58.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_36 2026-03-24T02:30:58.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:30:58.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=12 2026-03-24T02:30:58.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:30:58.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_36 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:30:59.616 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:59.611+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:30:59.616 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:59.611+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:30:59.617 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:59.611+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:30:59.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:59.615+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:30:59.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:59.615+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:30:59.668 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:30:59.663+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:30:59.719 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:30:59.715+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:31:02.697 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:02.691+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:31:02.697 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:02.691+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:31:03.654 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:31:03.645+0000 7f5496ce4640 -1 reset not still connected to 0x5652c8147d40 2026-03-24T02:31:03.657 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:31:03.649+0000 7f5496ce4640 -1 reset not still connected to 0x5652c92eb930 2026-03-24T02:31:03.657 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:31:03.649+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1466410 2026-03-24T02:31:03.660 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:31:03.653+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1e2a000 2026-03-24T02:31:03.660 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:31:03.653+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1e2b5f0 2026-03-24T02:31:03.660 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:31:03.653+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1e2bba0 2026-03-24T02:31:03.660 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:31:03.653+0000 7f5496ce4640 -1 reset not still connected to 0x5652ebdffd40 2026-03-24T02:31:03.882 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:03.875+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:31:03.883 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:03.879+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:31:04.617 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:04.611+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:31:04.617 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:04.611+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:31:04.623 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:04.619+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:31:04.623 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:04.619+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:31:04.624 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:04.619+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:31:04.671 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:04.667+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:31:04.720 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:04.715+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:31:04.949 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:31:04.941+0000 7fc159ee0640 -1 reset not still connected to 0x564185cc4820 2026-03-24T02:31:05.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 107732992 bytes 2026-03-24T02:31:08.656 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:08.651+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:31:08.656 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:08.651+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:31:08.657 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:08.651+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:31:08.797 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:08.791+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:31:08.797 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:08.791+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:31:08.797 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:08.791+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:31:08.797 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:08.791+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:31:08.797 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:08.791+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:31:08.797 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:08.791+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:31:08.799 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:08.795+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:31:08.882 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:08.875+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:31:08.883 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:08.879+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:31:09.619 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:09.611+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:31:09.622 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:09.615+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:31:09.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:09.619+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:31:09.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:09.619+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:31:09.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 163942400 bytes 2026-03-24T02:31:09.627 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:09.619+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:31:09.672 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:09.667+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:31:09.722 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:09.715+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:31:10.411 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:10.407+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:31:10.412 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:10.407+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_37 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_37 journal 128M 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_37 128M 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:31:10.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:10.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_37 2026-03-24T02:31:11.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:31:11.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:11.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:31:11.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:31:11.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:31:11.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:31:11.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_37 2026-03-24T02:31:11.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:31:11.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:11.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:31:11.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:31:11.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=9 2026-03-24T02:31:11.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:31:11.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_37 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:31:13.657 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:13.651+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:31:13.657 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:13.651+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:31:13.657 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:13.651+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:31:13.883 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:13.875+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:31:13.884 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:13.879+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:31:14.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:14.615+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:31:14.624 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:14.619+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:31:14.627 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:14.619+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:31:14.627 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:14.619+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:31:14.628 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:14.623+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8480 finish: resending after timeout 2026-03-24T02:31:14.673 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:14.667+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:31:14.723 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:14.715+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:31:15.295 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:15.291+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:31:15.295 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:15.291+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:31:15.297 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:15.291+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:31:15.298 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:15.291+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:31:15.299 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:15.295+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:31:15.300 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:15.295+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:31:15.300 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:15.295+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:31:15.412 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:15.407+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:31:15.414 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:15.407+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:31:15.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 44064768 bytes 2026-03-24T02:31:18.658 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:18.651+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e520c0 finish: resending after timeout 2026-03-24T02:31:18.658 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:18.651+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:31:18.658 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:18.651+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:31:18.884 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:18.879+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d39c6c0 finish: resending after timeout 2026-03-24T02:31:18.885 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:18.879+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:31:19.675 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:19.667+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_38 2026-03-24T02:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_38 journal 128M 2026-03-24T02:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_38 128M 2026-03-24T02:31:20.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:31:20.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:20.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:20.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:20.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:31:20.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:20.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:31:20.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:31:20.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:31:20.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:20.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_38 2026-03-24T02:31:20.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:31:20.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:20.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:31:20.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:31:20.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:31:20.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:31:20.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_38 2026-03-24T02:31:20.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:31:20.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:20.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:31:20.413 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:20.407+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:31:20.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:31:20.416 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:20.411+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:31:20.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=14 2026-03-24T02:31:20.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:31:20.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_38 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:31:23.887 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:23.879+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:31:24.679 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:24.675+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:31:24.988 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:24.983+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:31:24.988 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:24.983+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:31:24.988 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:24.983+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:31:24.988 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:24.983+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:31:24.988 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:24.983+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:31:24.988 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:24.983+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:31:24.990 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:24.983+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:31:25.010 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:25.003+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:31:25.418 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:25.411+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:31:25.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 52028416 bytes 2026-03-24T02:31:27.639 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:27.631+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:31:27.640 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:27.635+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:31:27.641 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:27.635+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:31:27.641 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:27.635+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:31:27.641 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:27.635+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:31:28.263 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:28.255+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:31:28.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 87027712 bytes 2026-03-24T02:31:28.886 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:28.879+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:31:29.680 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:29.675+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:31:30.011 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:30.007+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:31:31.923 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:31.915+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:31:31.925 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:31.919+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:31:32.297 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:32.291+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:31:32.298 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:32.291+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:31:32.300 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:32.295+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:31:32.300 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:32.295+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:31:32.300 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:32.295+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:31:32.300 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:32.295+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:31:32.300 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:32.295+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:31:32.701 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:32.695+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:31:32.701 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:32.695+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:31:32.701 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:32.695+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:31:32.701 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:32.695+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:31:32.702 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:32.695+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:31:32.702 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:32.695+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:31:33.264 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:33.259+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:31:33.886 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:33.879+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:31:34.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:31:34.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:31:34.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:31:34.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:31:34.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:31:34.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_39 2026-03-24T02:31:34.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_39 journal 128M 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_39 128M 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:34.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_39 2026-03-24T02:31:34.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:31:34.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:34.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:31:34.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:31:34.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:31:34.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:31:34.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_39 2026-03-24T02:31:34.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:31:34.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:34.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:31:34.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:31:34.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=9 2026-03-24T02:31:34.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:31:34.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_39 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:31:34.681 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:34.675+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:31:36.790 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:36.783+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:31:36.792 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:36.787+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:31:37.298 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:37.291+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:31:37.301 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:37.295+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:31:37.301 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:37.295+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:31:37.301 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:37.295+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:31:37.301 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:37.295+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:31:37.301 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:37.295+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:31:37.301 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:37.295+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:31:38.265 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:38.259+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:31:38.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 45469696 bytes 2026-03-24T02:31:38.887 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:38.879+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:31:39.682 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:39.675+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:31:41.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 54721536 bytes 2026-03-24T02:31:41.923 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:41.919+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:31:41.924 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:41.919+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:31:42.089 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:42.083+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:31:42.090 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:42.083+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:31:42.302 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:42.295+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:31:42.302 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:42.295+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:31:42.302 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:42.295+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:31:42.302 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:42.295+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:31:42.302 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:42.295+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:31:42.302 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:42.295+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:31:42.304 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:42.299+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:31:43.346 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:43.339+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_40 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_40 journal 128M 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_40 128M 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:43.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_40 2026-03-24T02:31:43.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:31:43.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:43.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:31:43.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:31:43.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:31:43.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:31:43.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_40 2026-03-24T02:31:43.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:31:43.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:43.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:31:43.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:31:43.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=7 2026-03-24T02:31:43.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:31:43.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_40 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:31:46.689 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:46.683+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:31:46.689 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:46.683+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:31:46.689 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:46.683+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:31:46.924 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:46.919+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:31:46.924 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:46.919+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:31:47.091 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:47.083+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:31:47.091 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:47.083+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:31:47.305 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:47.299+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:31:47.305 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:47.299+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:31:47.305 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:47.299+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:31:47.305 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:47.299+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:31:47.305 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:47.299+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:31:47.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:47.299+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:31:47.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:47.299+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:31:47.912 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:47.907+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:31:47.913 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:47.907+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:31:47.913 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:47.907+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:31:47.913 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:47.907+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:31:47.913 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:47.907+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:31:47.913 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:47.907+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:31:48.347 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:48.339+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:31:50.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:31:50.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:31:50.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:31:50.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:31:50.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:31:50.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_41 2026-03-24T02:31:50.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_41 journal 128M 2026-03-24T02:31:50.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_41 128M 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:31:50.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_41 2026-03-24T02:31:51.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:31:51.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:51.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:31:51.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:31:51.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:31:51.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:31:51.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_41 2026-03-24T02:31:51.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:31:51.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:31:51.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:31:51.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:31:51.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=9 2026-03-24T02:31:51.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:31:51.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_41 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:31:51.679 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:51.675+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:31:51.691 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:51.683+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:31:51.691 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:51.683+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:31:51.691 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:51.683+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:31:51.925 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:51.919+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:31:51.925 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:51.919+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:31:52.091 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:52.087+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:31:52.092 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:52.087+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:31:52.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:52.299+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:31:52.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:52.299+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:31:52.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:52.299+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:31:52.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:52.299+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:31:52.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:52.299+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:31:52.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:52.303+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:31:52.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:52.303+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:31:53.348 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:53.343+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:31:55.095 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:55.087+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:31:55.096 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:55.091+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:31:55.097 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:55.091+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:31:55.097 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:55.091+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:31:55.097 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:55.091+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:31:55.097 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:31:55.091+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:31:55.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 57044992 bytes 2026-03-24T02:31:55.392 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:31:55.385+0000 7fb2796c9640 -1 reset not still connected to 0x55ac15190ea0 2026-03-24T02:31:55.392 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:31:55.385+0000 7fb2796c9640 -1 reset not still connected to 0x55ac15254270 2026-03-24T02:31:55.392 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:31:55.385+0000 7fb2796c9640 -1 reset not still connected to 0x55ac152551e0 2026-03-24T02:31:55.392 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:31:55.385+0000 7fb2796c9640 -1 reset not still connected to 0x55ac15a72270 2026-03-24T02:31:55.392 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:31:55.385+0000 7fb2796c9640 -1 reset not still connected to 0x55ac15c3c680 2026-03-24T02:31:55.392 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:31:55.385+0000 7fb2796c9640 -1 reset not still connected to 0x55ac2d4ce0d0 2026-03-24T02:31:55.392 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:31:55.385+0000 7fb2796c9640 -1 reset not still connected to 0x55ac2d71b2b0 2026-03-24T02:31:55.392 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:31:55.385+0000 7fb2796c9640 -1 reset not still connected to 0x55ac2d71bc70 2026-03-24T02:31:55.783 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:31:55.777+0000 7fc159ee0640 -1 reset not still connected to 0x564185cc4820 2026-03-24T02:31:56.680 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:56.675+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:31:56.691 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:56.687+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:31:56.691 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:56.687+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:31:56.691 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:31:56.687+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:31:57.092 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:57.087+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:31:57.092 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:57.087+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:31:57.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:57.299+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:31:57.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:57.299+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:31:57.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:57.299+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:31:57.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:57.299+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:31:57.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:57.299+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:31:57.309 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:57.303+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:31:57.309 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:31:57.303+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:31:58.335 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.292781665617749) > weight (0.25) 2026-03-24T02:31:58.335 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.6228288093295298) > weight (0.25) 2026-03-24T02:31:58.335 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.468899494385302) > weight (0.25) 2026-03-24T02:31:58.335 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.4672373236974582) > weight (0.25) 2026-03-24T02:31:58.335 INFO:tasks.rbd_mirror_thrash:waiting for 94.72898424222925 secs before thrashing 2026-03-24T02:31:58.348 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:31:58.343+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:31:58.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 113128448 bytes 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_42 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_42 journal 128M 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_42 128M 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_42 2026-03-24T02:32:00.225 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:32:00.219+0000 7f085e5d7640 -1 reset not still connected to 0x558789dfa1a0 2026-03-24T02:32:00.225 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:32:00.219+0000 7f085e5d7640 -1 reset not still connected to 0x5587a614c340 2026-03-24T02:32:00.225 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:32:00.219+0000 7f085e5d7640 -1 reset not still connected to 0x5587a6ebe680 2026-03-24T02:32:00.225 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:32:00.219+0000 7f085e5d7640 -1 reset not still connected to 0x5587a6ebe9c0 2026-03-24T02:32:00.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:32:00.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:00.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:32:00.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:32:00.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:32:00.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:32:00.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_42 2026-03-24T02:32:00.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:00.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:00.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:32:00.230 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:32:00.223+0000 7f085e5d7640 -1 reset not still connected to 0x5587b5210c30 2026-03-24T02:32:00.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:32:00.231 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:32:00.223+0000 7f085e5d7640 -1 reset not still connected to 0x5587bb8da8f0 2026-03-24T02:32:00.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=5 2026-03-24T02:32:00.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:32:00.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_42 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:32:00.790 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:00.783+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:32:00.790 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:00.783+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:32:01.681 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:01.675+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:32:02.093 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:02.087+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:32:02.093 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:02.087+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:32:02.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:02.299+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:32:02.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:02.299+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:32:02.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:02.299+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:32:02.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:02.303+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:32:02.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:02.303+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:32:02.310 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:02.303+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:32:02.310 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:02.303+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:32:02.400 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:02.395+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:32:02.400 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:02.395+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:32:02.400 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:02.395+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:32:02.401 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:02.395+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:32:02.402 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:02.395+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:32:02.402 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:02.395+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:32:02.402 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:02.395+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:32:02.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 26446848 bytes 2026-03-24T02:32:03.350 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:03.343+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:32:05.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:32:05.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:32:05.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:32:05.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:32:05.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:32:05.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_43 2026-03-24T02:32:05.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_43 journal 128M 2026-03-24T02:32:05.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:05.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:05.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:05.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:05.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:32:05.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:05.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:32:05.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:32:05.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:05.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_43 128M 2026-03-24T02:32:05.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:05.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:05.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:05.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:05.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:32:05.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:05.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:32:05.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:32:05.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:32:05.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:05.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_43 2026-03-24T02:32:05.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:32:05.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:05.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:32:05.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:32:05.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:32:05.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:32:05.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_43 2026-03-24T02:32:05.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:05.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:05.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:32:05.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:32:05.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=9 2026-03-24T02:32:05.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:32:05.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_43 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:32:05.569 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:32:05.563+0000 7f773bdb3640 -1 reset not still connected to 0x5647b9975040 2026-03-24T02:32:05.603 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:05.595+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:32:05.603 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:05.599+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:32:05.604 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:05.599+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:32:06.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 4430848 bytes 2026-03-24T02:32:06.681 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:06.675+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:32:07.094 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:07.087+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:32:07.094 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:07.087+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:32:07.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:07.303+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:32:07.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:07.303+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:32:07.308 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:07.303+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:32:07.308 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:07.303+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:32:07.308 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:07.303+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:32:07.311 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:07.303+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:32:07.311 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:07.303+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:32:07.658 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:07.651+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:32:07.685 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:07.679+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:32:07.686 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:07.679+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:32:08.301 INFO:tasks.ceph.cluster2.osd.1.vm07.stderr:2026-03-24T02:32:08.295+0000 7f2e0c59f640 -1 reset not still connected to 0x56026ad12340 2026-03-24T02:32:08.309 INFO:tasks.ceph.cluster2.osd.1.vm07.stderr:2026-03-24T02:32:08.303+0000 7f2e0c59f640 -1 reset not still connected to 0x56026c60f110 2026-03-24T02:32:09.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 37076992 bytes 2026-03-24T02:32:10.070 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:10.063+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:32:10.070 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:10.063+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:32:10.071 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:10.063+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:32:10.071 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:10.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:32:10.073 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:10.067+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:32:10.074 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:10.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:32:10.075 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:10.071+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:32:10.604 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:10.599+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:32:10.604 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:10.599+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:32:10.606 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:10.599+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:32:11.682 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:11.675+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:32:12.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 61753344 bytes 2026-03-24T02:32:12.104 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:12.099+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:32:12.104 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:12.099+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:32:12.104 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:12.099+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:32:12.308 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:12.303+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:32:12.308 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:12.303+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:32:12.308 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:12.303+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:32:12.309 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:12.303+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:32:12.309 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:12.303+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:32:12.312 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:12.307+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:32:12.312 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:12.307+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:32:12.659 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:12.651+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:32:12.687 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:12.679+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:32:12.687 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:12.679+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:32:14.497 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:14.491+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:32:14.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:32:14.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:32:14.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:32:14.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:32:14.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:32:14.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_44 2026-03-24T02:32:14.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_44 journal 128M 2026-03-24T02:32:14.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:14.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:14.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:14.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:14.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:32:14.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:14.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:32:14.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:32:14.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:14.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_44 128M 2026-03-24T02:32:14.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:14.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:14.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:14.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:14.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:32:14.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:14.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:32:14.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:32:14.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:32:14.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:14.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_44 2026-03-24T02:32:14.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:32:14.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:14.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:32:14.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:32:14.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:32:14.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:32:14.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_44 2026-03-24T02:32:14.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:14.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:14.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:32:14.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:32:14.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=14 2026-03-24T02:32:14.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:32:14.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_44 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:32:15.071 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:15.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:32:15.072 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:15.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:32:15.072 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:15.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:32:15.072 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:15.067+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:32:15.074 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:15.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:32:15.075 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:15.067+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:32:15.076 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:15.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:32:15.605 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:15.599+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:32:15.605 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:15.599+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:32:15.606 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:15.599+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:32:16.683 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:16.675+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:32:17.104 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:17.099+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:32:17.104 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:17.099+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:32:17.104 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:17.099+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:32:17.310 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:17.303+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:32:17.310 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:17.303+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:32:17.310 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:17.303+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:32:17.310 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:17.303+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:32:17.310 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:17.303+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:32:17.312 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:17.307+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:32:17.312 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:17.307+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:32:17.660 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:17.655+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:32:19.498 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:19.491+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:32:20.073 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:20.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:32:20.073 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:20.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:32:20.073 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:20.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:32:20.073 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:20.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2480 finish: resending after timeout 2026-03-24T02:32:20.075 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:20.067+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:32:20.075 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:20.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:32:20.077 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:20.071+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:32:20.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 77479936 bytes 2026-03-24T02:32:20.606 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:20.599+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:32:20.606 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:20.599+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:32:20.606 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:20.599+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:32:21.687 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:21.679+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:32:21.818 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:21.811+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:32:21.818 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:21.811+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:32:22.105 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:22.099+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:32:22.105 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:22.099+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:32:22.105 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:22.099+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:32:22.311 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:22.303+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:32:22.311 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:22.303+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:32:22.311 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:22.303+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:32:22.312 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:22.307+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:32:22.313 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:22.307+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:32:22.313 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:22.307+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:32:22.313 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:22.307+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:32:22.660 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:22.655+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:32:23.148 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:23.143+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:32:23.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 115585024 bytes 2026-03-24T02:32:24.499 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:24.491+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:32:25.073 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:25.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:32:25.074 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:25.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:32:25.074 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:25.067+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:32:25.075 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:25.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:32:25.075 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:25.071+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:32:25.078 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:25.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:32:25.606 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:25.599+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:32:25.607 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:25.599+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:32:25.607 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:25.599+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:32:26.686 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:26.679+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:32:26.818 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:26.811+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:32:27.107 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:27.099+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:32:27.107 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:27.099+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:32:27.107 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:27.099+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:32:27.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 156034048 bytes 2026-03-24T02:32:27.314 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:27.307+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:32:27.314 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:27.307+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:32:27.314 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:27.307+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:32:27.314 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:27.307+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:32:27.314 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:27.307+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:32:27.314 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:27.307+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:32:27.314 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:27.307+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:32:27.661 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:27.655+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:32:28.149 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:28.143+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:32:28.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:32:28.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:32:28.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_45 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_45 journal 128M 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_45 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_45 128M 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_45 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:28.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_45 2026-03-24T02:32:29.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:32:29.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:29.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:32:29.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:32:29.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:32:29.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:32:29.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_45 2026-03-24T02:32:29.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:29.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:29.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_45 2026-03-24T02:32:29.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:32:29.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=13 2026-03-24T02:32:29.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:32:29.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_45 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:32:29.500 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:29.495+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:32:30.029 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:30.023+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:32:30.075 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:30.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:32:30.076 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:30.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:32:30.076 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:30.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:32:30.078 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:30.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:32:30.079 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:30.071+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:32:30.079 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:30.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:32:30.607 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:30.599+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:32:30.607 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:30.599+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:32:30.608 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:30.603+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:32:31.687 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:31.679+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:32:31.819 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:31.811+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:32:31.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 24602624 bytes 2026-03-24T02:32:32.107 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:32.099+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:32:32.107 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:32.099+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:32:32.107 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:32.103+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:32:32.315 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:32.307+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:32:32.315 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:32.307+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:32:32.315 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:32.307+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:32:32.315 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:32.307+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:32:32.315 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:32.311+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:32:32.316 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:32.311+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:32:32.316 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:32.311+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:32:32.662 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:32.655+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:32:32.890 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:32:32.883+0000 7f085e5d7640 -1 reset not still connected to 0x558789dfa1a0 2026-03-24T02:32:32.894 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:32:32.887+0000 7f085e5d7640 -1 reset not still connected to 0x5587a614c340 2026-03-24T02:32:32.894 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:32:32.887+0000 7f085e5d7640 -1 reset not still connected to 0x5587a6ebe680 2026-03-24T02:32:32.894 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:32:32.887+0000 7f085e5d7640 -1 reset not still connected to 0x5587a6ebe9c0 2026-03-24T02:32:32.894 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:32:32.887+0000 7f085e5d7640 -1 reset not still connected to 0x5587b5210c30 2026-03-24T02:32:32.894 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:32:32.887+0000 7f085e5d7640 -1 reset not still connected to 0x5587bb8da8f0 2026-03-24T02:32:33.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 33298432 bytes 2026-03-24T02:32:33.151 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:33.143+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:32:33.222 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7756126909814937) > weight (0.25) 2026-03-24T02:32:33.222 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7313890299285088) > weight (0.25) 2026-03-24T02:32:33.222 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-03-24T02:32:33.222 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-03-24T02:32:33.222 INFO:tasks.rbd_mirror_thrash:waiting for 1.2234827772401735 secs before reviving daemons 2026-03-24T02:32:33.223 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:32:33.215+0000 7f621f45e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 38686) UID: 1000 2026-03-24T02:32:34.446 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-03-24T02:32:34.446 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:32:34.500 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:34.495+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:32:35.029 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:35.023+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:32:35.075 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:35.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:32:35.075 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:35.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:32:35.075 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:35.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:32:35.078 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:35.071+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:32:35.078 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:35.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:32:35.079 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:35.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:32:35.608 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:35.603+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:32:35.608 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:35.603+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:32:35.608 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:35.603+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:32:36.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 76398592 bytes 2026-03-24T02:32:36.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:36.683+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:32:36.819 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:36.815+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:32:36.826 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:32:36.819+0000 7f773bdb3640 -1 reset not still connected to 0x5647d0992820 2026-03-24T02:32:37.030 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:37.023+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:32:37.108 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:37.103+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:32:37.108 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:37.103+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:32:37.108 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:37.103+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:32:37.316 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:37.311+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:32:37.316 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:37.311+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:32:37.316 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:37.311+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:32:37.316 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:37.311+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:32:37.316 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:37.311+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:32:37.316 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:37.311+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:32:37.316 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:37.311+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:32:37.662 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:37.655+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:32:38.151 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:38.143+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:32:39.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:400 IOs, 124441600 bytes 2026-03-24T02:32:39.502 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:39.495+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:32:40.030 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:40.023+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:32:40.076 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:40.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:32:40.076 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:40.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:32:40.076 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:40.071+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:32:40.079 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:40.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:32:40.079 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:40.071+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:32:40.079 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:40.075+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:32:40.547 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-03-24T02:32:40.547 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-03-24T02:32:40.547 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-03-24T02:32:40.549 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-03-24T02:32:40.549 INFO:tasks.rbd_mirror_thrash:waiting for 71.38731290275635 secs before thrashing 2026-03-24T02:32:40.609 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:40.603+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:32:40.609 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:40.603+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:32:40.609 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:40.603+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b440 finish: resending after timeout 2026-03-24T02:32:41.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:41.683+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:32:41.820 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:41.815+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:32:42.031 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:42.023+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:32:42.108 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:42.103+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:32:42.109 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:42.103+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:32:42.109 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:42.103+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:32:42.316 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:42.311+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:32:42.316 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:42.311+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:32:42.316 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:42.311+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:32:42.316 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:42.311+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:32:42.317 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:42.311+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:32:42.317 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:42.311+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:32:42.317 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:42.311+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:32:42.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:500 IOs, 155044864 bytes 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_46 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_46 journal 128M 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_46 128M 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:42.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_46 2026-03-24T02:32:42.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:32:42.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:42.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:32:42.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:32:42.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:32:42.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:32:42.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_46 2026-03-24T02:32:42.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:42.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:42.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:32:42.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:32:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=14 2026-03-24T02:32:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:32:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_46 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:32:42.663 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:42.655+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:32:43.152 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:43.147+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:32:44.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 9479168 bytes 2026-03-24T02:32:44.503 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:44.495+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:32:45.030 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:45.023+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:32:45.076 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:45.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:32:45.077 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:45.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:32:45.077 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:45.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:32:45.079 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:45.075+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:32:45.080 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:45.075+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:32:45.610 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:45.603+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:32:45.610 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:45.603+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:32:46.821 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:46.815+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:32:47.032 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:47.027+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:32:47.109 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:47.103+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:32:47.109 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:47.103+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:32:47.109 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:47.103+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:32:47.358 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:47.351+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:32:47.664 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:47.659+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:32:48.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 57176064 bytes 2026-03-24T02:32:50.031 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:50.023+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:32:50.077 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:50.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:32:50.077 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:50.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:32:50.077 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:50.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:32:50.080 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:50.075+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:32:50.081 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:50.075+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:32:50.548 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:50.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:32:50.548 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:50.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:32:50.548 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:50.543+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:32:50.548 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:50.543+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:32:50.548 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:50.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:32:50.548 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:50.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:32:50.548 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:50.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:32:50.548 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:50.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:32:50.548 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:50.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:32:50.583 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:50.579+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:32:50.611 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:50.603+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:32:50.613 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:50.607+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:32:51.821 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:51.815+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:32:52.032 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:52.027+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:32:52.359 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:52.351+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:32:52.664 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:52.659+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:32:52.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 109283328 bytes 2026-03-24T02:32:55.032 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:55.027+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:32:55.079 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:55.071+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:32:55.079 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:55.071+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:32:55.079 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:55.071+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:32:55.080 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:55.075+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:32:55.082 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:55.075+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:32:55.548 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:55.543+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:32:55.548 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:55.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:32:55.548 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:55.543+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:32:55.549 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:55.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:32:55.549 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:55.543+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:32:55.549 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:55.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:32:55.549 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:55.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:32:55.549 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:55.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:32:55.549 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:32:55.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:32:55.586 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:55.579+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:32:55.611 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:55.603+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:32:55.612 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:55.607+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_47 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_47 journal 128M 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_47 128M 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:56.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:56.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:32:56.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:56.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:32:56.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:32:56.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:32:56.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:32:56.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_47 2026-03-24T02:32:56.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:32:56.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:56.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:32:56.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:32:56.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:32:56.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:32:56.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_47 2026-03-24T02:32:56.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:32:56.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:32:56.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:32:56.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:32:56.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=14 2026-03-24T02:32:56.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:32:56.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_47 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:32:56.822 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:56.815+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:32:57.033 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:57.027+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:32:57.359 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:32:57.355+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:32:57.665 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:32:57.659+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:32:57.714 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:57.707+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:32:57.714 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:57.707+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:32:57.714 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:57.707+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:32:57.714 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:32:57.707+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:33:00.032 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:00.027+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:33:00.082 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:00.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:33:00.082 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:00.075+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:33:00.082 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:00.075+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:33:00.082 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:00.075+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:33:00.082 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:00.075+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:33:00.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:00.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:33:00.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:00.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:33:00.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:00.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:33:00.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:00.543+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:33:00.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:00.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:33:00.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:00.543+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:33:00.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:00.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:33:00.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:00.543+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:33:00.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:00.543+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:33:00.586 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:00.579+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:33:00.612 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:00.607+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:33:00.612 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:00.607+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:33:01.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 44724224 bytes 2026-03-24T02:33:01.822 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:01.815+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:33:02.033 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:02.027+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:33:02.361 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:02.355+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:33:02.666 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:02.659+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:33:02.715 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:02.707+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:33:02.715 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:02.707+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:33:02.715 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:02.707+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:33:02.725 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:02.719+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda49c0 finish: resending after timeout 2026-03-24T02:33:04.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 81224704 bytes 2026-03-24T02:33:05.033 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:05.027+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:33:05.080 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:05.075+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:33:05.080 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:05.075+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:33:05.083 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:05.075+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:33:05.083 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:05.075+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:33:05.549 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:05.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:33:05.573 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:05.543+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:33:05.573 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:05.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:33:05.573 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:05.543+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:33:05.573 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:05.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:33:05.573 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:05.543+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:33:05.573 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:05.543+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:33:05.573 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:05.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:33:05.573 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:05.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:33:05.589 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:05.583+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:33:05.613 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:05.607+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:33:05.643 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:05.607+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:33:05.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 89509888 bytes 2026-03-24T02:33:06.316 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:06.311+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:33:06.823 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:06.815+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:33:06.838 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:33:06.829+0000 7f5496ce4640 -1 reset not still connected to 0x5652c68abba0 2026-03-24T02:33:06.845 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:33:06.837+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f9860 2026-03-24T02:33:06.845 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:33:06.837+0000 7f5496ce4640 -1 reset not still connected to 0x5652c8147d40 2026-03-24T02:33:06.846 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:33:06.841+0000 7f5496ce4640 -1 reset not still connected to 0x5652c92eab60 2026-03-24T02:33:06.846 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:33:06.841+0000 7f5496ce4640 -1 reset not still connected to 0x5652c92eb930 2026-03-24T02:33:06.847 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:33:06.841+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1466410 2026-03-24T02:33:06.847 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:33:06.841+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1e2a000 2026-03-24T02:33:06.847 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:33:06.841+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1e2b5f0 2026-03-24T02:33:06.854 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:33:06.845+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1e2bba0 2026-03-24T02:33:06.854 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:33:06.845+0000 7f5496ce4640 -1 reset not still connected to 0x5652ebdffd40 2026-03-24T02:33:07.035 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:07.031+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:33:07.669 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:07.663+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:33:08.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:400 IOs, 110059520 bytes 2026-03-24T02:33:09.205 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:09.199+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:33:10.033 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:10.027+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:33:10.081 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:10.075+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:33:10.081 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:10.075+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:33:10.083 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:10.079+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:33:10.083 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:10.079+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:33:10.550 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:10.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:33:10.550 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:10.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:33:10.550 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:10.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:33:10.550 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:10.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:33:10.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:10.543+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:33:10.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:10.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:33:10.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:10.543+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:33:10.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:10.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:33:10.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:10.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:33:10.589 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:10.583+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:33:10.614 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:10.607+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:33:10.614 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:10.607+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_48 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_48 journal 128M 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_48 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_48 128M 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_48 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:10.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_48 2026-03-24T02:33:11.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:33:11.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:33:11.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:33:11.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:33:11.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:33:11.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:33:11.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_48 2026-03-24T02:33:11.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:33:11.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:33:11.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_48 2026-03-24T02:33:11.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:33:11.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=10 2026-03-24T02:33:11.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:33:11.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_48 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:33:12.013 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:33:12.005+0000 7fc159ee0640 -1 reset not still connected to 0x56417f827930 2026-03-24T02:33:12.040 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:12.035+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:33:12.190 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:12.183+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:33:12.190 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:12.183+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:33:12.669 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:12.663+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:33:14.206 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:14.199+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:33:14.707 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:14.703+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:33:14.708 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:14.703+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:33:14.708 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:14.703+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:33:14.708 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:14.703+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:33:15.035 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:15.027+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:33:15.082 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:15.075+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:33:15.082 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:15.075+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:33:15.084 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:15.079+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:33:15.084 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:15.079+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:33:15.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:15.543+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:33:15.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:15.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:33:15.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:15.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:33:15.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:15.543+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:33:15.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:15.543+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:33:15.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:15.547+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:33:15.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:15.547+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:33:15.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:15.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:33:15.551 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:15.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:33:15.590 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:15.583+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:33:15.615 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:15.607+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:33:15.615 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:15.607+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:33:17.040 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:17.035+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:33:17.190 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:17.183+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:33:17.192 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:17.183+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:33:17.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 55532544 bytes 2026-03-24T02:33:17.670 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:17.663+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:33:18.323 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:18.315+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:33:18.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 71682048 bytes 2026-03-24T02:33:19.207 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:19.199+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:33:20.036 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:20.031+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:33:20.083 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:20.075+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:33:20.083 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:20.079+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:33:20.085 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:20.079+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:33:20.085 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:20.079+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:33:20.552 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:20.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:33:20.552 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:20.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:33:20.552 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:20.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:33:20.552 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:20.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:33:20.552 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:20.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:33:20.552 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:20.547+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:33:20.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:20.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:33:20.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:20.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:33:20.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:20.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:33:20.590 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:20.583+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:33:20.616 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:20.611+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:33:20.616 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:20.611+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:33:21.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:33:21.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:33:21.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:33:21.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_49 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_49 journal 128M 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_49 128M 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:21.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_49 2026-03-24T02:33:21.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:33:21.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:33:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:33:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:33:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:33:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:33:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_49 2026-03-24T02:33:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:33:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:33:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:33:21.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:33:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=11 2026-03-24T02:33:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:33:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_49 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:33:22.041 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:22.035+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:33:22.671 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:22.663+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:33:22.969 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:22.963+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:33:22.969 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:22.963+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:33:22.969 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:22.963+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:33:22.969 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:22.963+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:33:23.324 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:23.319+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:33:24.209 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:24.203+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:33:25.038 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:25.031+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:33:25.084 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:25.079+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:33:25.084 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:25.079+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:33:25.085 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:25.079+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:33:25.086 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:25.079+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:33:25.553 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:25.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:33:25.553 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:25.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:33:25.553 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:25.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:33:25.554 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:25.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:33:25.554 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:25.547+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:33:25.554 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:25.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:33:25.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:25.551+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:33:25.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:25.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:33:25.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:25.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:33:25.591 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:25.583+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:33:25.629 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:25.611+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:33:25.629 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:25.611+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:33:26.841 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:26.835+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:33:26.841 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:26.835+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:33:27.044 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:27.035+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:33:27.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 58657792 bytes 2026-03-24T02:33:27.671 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:27.667+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:33:27.969 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:27.963+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:33:27.969 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:27.963+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:33:27.969 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:27.963+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:33:27.969 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:27.963+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:33:28.326 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:28.319+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:33:28.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 76240896 bytes 2026-03-24T02:33:29.209 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:29.203+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:33:30.038 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:30.031+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:33:30.085 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:30.079+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:33:30.085 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:30.079+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:33:30.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:30.079+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:33:30.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:30.079+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:33:30.553 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:30.547+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:33:30.554 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:30.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:33:30.554 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:30.547+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:33:30.554 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:30.547+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:33:30.554 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:30.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:33:30.554 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:30.547+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:33:30.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:30.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:33:30.556 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:30.551+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:33:30.558 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:30.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:33:30.591 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:30.587+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:33:30.616 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:30.611+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:33:30.616 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:30.611+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:33:31.841 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:31.835+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:33:31.842 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:31.835+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:33:32.043 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:32.035+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:33:32.063 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:32.055+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:33:32.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:33:32.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:33:32.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:33:32.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:33:32.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:33:32.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_50 2026-03-24T02:33:32.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image_and_enable_mirror cluster2 mirror image_50 journal 128M 2026-03-24T02:33:32.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:33:32.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:32.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:33:32.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:32.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:33:32.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:32.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:33:32.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n journal ']' 2026-03-24T02:33:32.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:32.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_50 128M 2026-03-24T02:33:32.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:33:32.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:32.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:33:32.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:32.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:33:32.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:32.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:33:32.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 128M ']' 2026-03-24T02:33:32.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=128M 2026-03-24T02:33:32.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:33:32.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_50 2026-03-24T02:33:32.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ is_pool_mirror_mode_image mirror 2026-03-24T02:33:32.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:33:32.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' pool = image ']' 2026-03-24T02:33:32.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ case "${pool}" in 2026-03-24T02:33:32.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 1 2026-03-24T02:33:32.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n '' ']' 2026-03-24T02:33:32.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_50 2026-03-24T02:33:32.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:33:32.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:33:32.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:33:32.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:33:32.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=12 2026-03-24T02:33:32.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:33:32.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_50 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:33:32.672 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:32.667+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:33:33.065 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.3805589918840948) > weight (0.25) 2026-03-24T02:33:33.065 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.8837369364215127) > weight (0.25) 2026-03-24T02:33:33.065 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.8188910492872329) > weight (0.25) 2026-03-24T02:33:33.065 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.4353962504439213) > weight (0.25) 2026-03-24T02:33:33.065 INFO:tasks.rbd_mirror_thrash:waiting for 98.16823916110465 secs before thrashing 2026-03-24T02:33:33.326 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:33.319+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:33:33.811 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:33.803+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:33:33.811 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:33.803+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:33:33.811 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:33.803+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:33:33.811 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:33.803+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:33:34.210 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:34.203+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:33:34.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 18130944 bytes 2026-03-24T02:33:35.039 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:35.031+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:33:35.085 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:35.079+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:33:35.085 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:35.079+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:33:35.086 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:35.079+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:33:35.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:35.079+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:33:35.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:35.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:33:35.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:35.547+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:33:35.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:35.547+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:33:35.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:35.547+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:33:35.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:35.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:33:35.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:35.551+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:33:35.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:35.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:33:35.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:35.551+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:33:35.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:35.555+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:33:35.593 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:35.587+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:33:35.617 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:35.611+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:33:35.617 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:35.611+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:33:35.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 23705600 bytes 2026-03-24T02:33:36.842 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:36.835+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:33:36.851 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:36.843+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:33:37.043 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:37.035+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:33:37.066 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:37.059+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:33:37.672 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:37.667+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:33:38.327 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:38.319+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:33:38.812 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:38.807+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:33:38.812 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:38.807+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:33:38.812 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:38.807+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:33:38.812 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:38.807+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:33:39.212 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:39.207+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:33:39.819 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:39.811+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:33:40.042 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:40.035+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:33:40.086 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:40.079+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:33:40.086 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:40.079+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:33:40.086 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:40.079+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:33:40.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:40.083+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:33:40.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:40.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:33:40.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:40.551+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:33:40.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:40.551+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:33:40.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:40.551+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:33:40.556 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:40.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:33:40.556 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:40.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:33:40.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:40.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:33:40.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:40.555+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:33:40.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:40.555+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:33:40.594 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:40.587+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:33:40.619 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:40.611+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:33:40.619 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:40.611+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:33:40.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 84069376 bytes 2026-03-24T02:33:41.844 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:41.839+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:33:41.851 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:41.843+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:33:42.044 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:42.039+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:33:42.071 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:42.067+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:33:42.675 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:42.667+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:33:43.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:400 IOs, 103537664 bytes 2026-03-24T02:33:43.327 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:43.319+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:33:43.812 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:43.807+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:33:43.812 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:43.807+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:33:43.813 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:43.807+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:33:43.813 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:43.807+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:33:44.213 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:44.207+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:33:44.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:33:44.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:33:44.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:33:44.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:33:44.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_pool_images cluster2 mirror 50 2026-03-24T02:33:44.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:33:44.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:33:44.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_count=50 2026-03-24T02:33:44.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:33:44.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local count 2026-03-24T02:33:44.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_count=0 2026-03-24T02:33:44.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:33:44.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 1 40 2026-03-24T02:33:44.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in `seq 1 40` 2026-03-24T02:33:44.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 1 -ne 1 2026-03-24T02:33:44.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-03-24T02:33:44.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep 'images: ' 2026-03-24T02:33:44.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut '-d ' -f 2 2026-03-24T02:33:44.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: failed to query services: (13) Permission denied 2026-03-24T02:33:44.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ count=50 2026-03-24T02:33:44.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 50 = 50 2026-03-24T02:33:44.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:33:44.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_pool_healthy cluster2 mirror 2026-03-24T02:33:44.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:33:44.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:33:44.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:33:44.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state 2026-03-24T02:33:44.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 1 40 2026-03-24T02:33:44.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in `seq 1 40` 2026-03-24T02:33:44.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 1 -ne 1 2026-03-24T02:33:44.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-03-24T02:33:44.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep 'image health:' 2026-03-24T02:33:44.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut '-d ' -f 3 2026-03-24T02:33:44.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: failed to query services: (13) Permission denied 2026-03-24T02:33:44.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=WARNING 2026-03-24T02:33:44.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test WARNING = ERROR 2026-03-24T02:33:44.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test WARNING = OK 2026-03-24T02:33:44.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in `seq 1 40` 2026-03-24T02:33:44.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 2 -ne 1 2026-03-24T02:33:44.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 30 2026-03-24T02:33:44.819 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:44.815+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:33:45.043 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:45.035+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:33:45.086 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:45.079+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:33:45.086 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:45.079+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:33:45.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:45.079+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:33:45.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:45.083+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:33:45.556 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:45.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:33:45.556 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:45.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:33:45.556 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:45.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:33:45.556 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:45.551+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:33:45.556 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:45.551+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:33:45.556 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:45.551+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:33:45.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:45.555+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:33:45.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:45.555+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:33:45.560 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:45.555+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:33:45.594 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:45.587+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:33:45.619 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:45.615+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:33:45.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:45.615+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:33:46.216 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:33:46.211+0000 7f085e5d7640 -1 reset not still connected to 0x5587838c7790 2026-03-24T02:33:46.844 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:46.839+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:33:46.851 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:46.847+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:33:47.045 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:47.039+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:33:47.076 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:47.071+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:33:47.674 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:47.667+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:33:48.328 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:48.323+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:33:48.813 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:48.807+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:33:48.813 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:48.807+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:33:48.813 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:48.807+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:33:48.813 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:48.807+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:33:49.214 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:49.207+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:33:49.821 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:49.815+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:33:50.043 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:50.039+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:33:50.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:50.079+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:33:50.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:50.079+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:33:50.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:50.083+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:33:50.088 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:50.083+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:33:50.558 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:50.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:33:50.558 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:50.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:33:50.558 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:50.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:33:50.558 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:50.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:33:50.558 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:50.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:33:50.558 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:50.551+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:33:50.560 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:50.555+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636f90bbe00 finish: resending after timeout 2026-03-24T02:33:50.560 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:50.555+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:33:50.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:50.555+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:33:50.594 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:50.587+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:33:50.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:50.615+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:33:50.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:50.615+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:33:51.845 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:51.840+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:33:51.852 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:51.848+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:33:51.936 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8475584208878346) > weight (0.25) 2026-03-24T02:33:51.937 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-03-24T02:33:51.937 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-03-24T02:33:51.937 INFO:tasks.rbd_mirror_thrash:waiting for 5.232753781036593 secs before reviving daemons 2026-03-24T02:33:51.937 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:33:51.932+0000 7f20bb32e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 46976) UID: 1000 2026-03-24T02:33:52.046 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:52.040+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:33:52.078 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:52.072+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:33:52.675 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:52.668+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:33:53.329 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:53.324+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:33:53.814 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:53.808+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:33:53.814 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:53.808+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:33:53.814 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:53.808+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:33:53.814 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:53.808+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:33:54.821 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:54.816+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:33:55.558 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:55.552+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:33:55.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:55.556+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:33:55.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:55.556+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ecc00 finish: resending after timeout 2026-03-24T02:33:55.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:55.556+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fae9a9c0 finish: resending after timeout 2026-03-24T02:33:55.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:55.556+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:33:55.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:55.556+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:33:55.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:55.556+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:33:55.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:55.560+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:33:57.170 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-03-24T02:33:57.170 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:33:57.170 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-03-24T02:33:57.170 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-03-24T02:33:57.170 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-03-24T02:33:57.171 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-03-24T02:33:57.171 INFO:tasks.rbd_mirror_thrash:waiting for 110.35251880300922 secs before thrashing 2026-03-24T02:33:57.321 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:57.316+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:33:58.286 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:58.280+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:33:58.286 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:58.280+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:33:58.286 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:58.280+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:33:58.286 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:58.280+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:33:58.286 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:58.280+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:33:58.286 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:58.280+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:33:58.286 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:58.280+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:33:58.330 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:33:58.324+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:33:58.814 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:58.808+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:33:58.815 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:58.808+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:33:58.815 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:58.808+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:33:58.815 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:33:58.808+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:33:58.826 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:33:58.820+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:33:59.831 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:59.824+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:33:59.831 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:59.824+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:33:59.831 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:59.824+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764da17680 finish: resending after timeout 2026-03-24T02:33:59.831 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:59.824+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:33:59.831 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:33:59.824+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:34:00.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:00.556+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:34:00.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:00.556+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:34:00.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:00.556+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:34:00.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:00.556+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:34:00.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:00.556+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:34:00.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:00.560+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:34:00.899 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:00.889+0000 7f5496ce4640 -1 reset not still connected to 0x5652c95dc8f0 2026-03-24T02:34:00.899 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:00.893+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c31380 2026-03-24T02:34:00.899 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:00.893+0000 7f5496ce4640 -1 reset not still connected to 0x5652d23d2d00 2026-03-24T02:34:00.899 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:00.893+0000 7f5496ce4640 -1 reset not still connected to 0x5652d23d31e0 2026-03-24T02:34:00.900 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:00.897+0000 7f5496ce4640 -1 reset not still connected to 0x5652ebdff2b0 2026-03-24T02:34:03.288 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:03.284+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:34:03.288 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:03.284+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:34:03.288 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:03.284+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:34:03.288 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:03.284+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:34:03.288 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:03.284+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:34:03.330 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:03.324+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:34:03.815 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:03.812+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:34:03.815 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:03.812+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:34:03.819 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:03.812+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:34:03.819 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:03.812+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:34:03.828 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:03.824+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:34:04.728 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:34:04.720+0000 7f773bdb3640 -1 reset not still connected to 0x5647c0cbdd40 2026-03-24T02:34:05.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:05.556+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:34:05.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:05.556+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:34:05.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:05.556+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:34:05.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:05.556+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:34:05.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:05.556+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:34:05.565 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:05.560+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:34:05.735 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:34:05.725+0000 7fb2796c9640 -1 reset not still connected to 0x55ac138aec30 2026-03-24T02:34:05.736 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:34:05.729+0000 7fb2796c9640 -1 reset not still connected to 0x55ac138afc70 2026-03-24T02:34:05.736 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:34:05.729+0000 7fb2796c9640 -1 reset not still connected to 0x55ac143176c0 2026-03-24T02:34:05.737 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:34:05.729+0000 7fb2796c9640 -1 reset not still connected to 0x55ac15254750 2026-03-24T02:34:05.738 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:34:05.729+0000 7fb2796c9640 -1 reset not still connected to 0x55ac15c3d110 2026-03-24T02:34:05.739 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:34:05.733+0000 7fb2796c9640 -1 reset not still connected to 0x55ac15c3d930 2026-03-24T02:34:05.739 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:34:05.733+0000 7fb2796c9640 -1 reset not still connected to 0x55ac3975edd0 2026-03-24T02:34:05.892 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:05.888+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:34:05.895 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:05.892+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:34:05.896 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:05.892+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:34:05.896 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:05.892+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:34:08.331 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:08.328+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:34:08.816 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:08.812+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:34:08.816 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:08.812+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:34:08.819 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:08.816+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:34:08.819 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:08.816+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:34:10.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:10.556+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:34:10.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:10.560+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:34:10.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:10.560+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:34:10.564 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:10.560+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:34:10.564 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:10.560+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:34:10.565 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:10.560+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:34:10.737 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:10.732+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:34:10.737 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:10.732+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:34:10.737 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:10.732+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:34:11.049 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:11.044+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:34:11.049 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:11.044+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:34:11.049 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:11.044+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:34:11.049 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:11.044+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:34:11.051 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:11.044+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:34:11.345 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:11.337+0000 7fc159ee0640 -1 reset not still connected to 0x5641840be270 2026-03-24T02:34:11.348 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:11.341+0000 7fc159ee0640 -1 reset not still connected to 0x5641a16cec30 2026-03-24T02:34:11.803 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:34:11.797+0000 7fb2796c9640 -1 reset not still connected to 0x55ac152551e0 2026-03-24T02:34:11.803 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:34:11.797+0000 7fb2796c9640 -1 reset not still connected to 0x55ac15a72270 2026-03-24T02:34:12.485 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:12.477+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f9380 2026-03-24T02:34:13.332 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:13.328+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:34:13.816 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:13.812+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:34:13.816 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:13.812+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:34:13.822 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:13.816+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:34:13.822 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:13.816+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:34:14.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut '-d ' -f 3 2026-03-24T02:34:14.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-03-24T02:34:14.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep 'image health:' 2026-03-24T02:34:14.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: failed to query services: (13) Permission denied 2026-03-24T02:34:14.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=WARNING 2026-03-24T02:34:14.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test WARNING = ERROR 2026-03-24T02:34:14.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test WARNING = OK 2026-03-24T02:34:14.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in `seq 1 40` 2026-03-24T02:34:14.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 -ne 1 2026-03-24T02:34:14.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 30 2026-03-24T02:34:15.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:15.560+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:34:15.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:15.560+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:34:15.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:15.560+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:34:15.564 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:15.560+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:34:15.565 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:15.560+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:34:15.566 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:15.560+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ec0c0 finish: resending after timeout 2026-03-24T02:34:15.738 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:15.732+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:34:15.738 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:15.732+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:34:15.738 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:15.732+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:34:15.769 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:15.764+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:34:15.769 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:15.764+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:34:16.051 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:16.044+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:34:16.051 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:16.044+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:34:16.055 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:16.048+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2780 finish: resending after timeout 2026-03-24T02:34:16.055 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:16.048+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:34:16.055 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:16.048+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:34:16.108 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:34:16.104+0000 7f085e5d7640 -1 reset not still connected to 0x5587898835f0 2026-03-24T02:34:16.114 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:34:16.108+0000 7f085e5d7640 -1 reset not still connected to 0x55879504cc30 2026-03-24T02:34:16.114 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:34:16.108+0000 7f085e5d7640 -1 reset not still connected to 0x5587a1fe95f0 2026-03-24T02:34:16.114 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:34:16.108+0000 7f085e5d7640 -1 reset not still connected to 0x5587b5213860 2026-03-24T02:34:17.108 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:17.101+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c31ad0 2026-03-24T02:34:17.117 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:17.109+0000 7f5496ce4640 -1 reset not still connected to 0x5652d23d25b0 2026-03-24T02:34:17.117 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:17.109+0000 7f5496ce4640 -1 reset not still connected to 0x5652d23d2f70 2026-03-24T02:34:17.117 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:17.109+0000 7f5496ce4640 -1 reset not still connected to 0x5652d5c3f5f0 2026-03-24T02:34:17.117 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:17.109+0000 7f5496ce4640 -1 reset not still connected to 0x5652ebdffba0 2026-03-24T02:34:19.318 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:19.312+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:34:19.318 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:19.312+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:34:19.318 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:19.312+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:34:19.318 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:19.312+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:34:20.738 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:20.732+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:34:20.741 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:20.736+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:34:20.741 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:20.736+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:34:20.770 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:20.764+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:34:21.053 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:21.048+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:34:21.053 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:21.048+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:34:21.056 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:21.052+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:34:21.057 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:21.052+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:34:21.100 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:21.096+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:34:21.100 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:21.096+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:34:21.100 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:21.096+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:34:21.100 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:21.096+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:34:22.109 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:22.104+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:34:22.110 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:22.104+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:34:22.110 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:22.104+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed500 finish: resending after timeout 2026-03-24T02:34:22.110 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:22.104+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:34:22.110 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:22.104+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:34:22.110 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:22.104+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:34:23.742 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:23.733+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f95f0 2026-03-24T02:34:23.745 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:23.737+0000 7f5496ce4640 -1 reset not still connected to 0x5652c7c3bc70 2026-03-24T02:34:23.745 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:23.737+0000 7f5496ce4640 -1 reset not still connected to 0x5652c92ea4e0 2026-03-24T02:34:23.745 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:23.737+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c31ad0 2026-03-24T02:34:23.745 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:23.737+0000 7f5496ce4640 -1 reset not still connected to 0x5652d23d25b0 2026-03-24T02:34:23.745 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:23.737+0000 7f5496ce4640 -1 reset not still connected to 0x5652d23d2f70 2026-03-24T02:34:23.745 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:23.737+0000 7f5496ce4640 -1 reset not still connected to 0x5652d5c3f5f0 2026-03-24T02:34:23.745 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:23.737+0000 7f5496ce4640 -1 reset not still connected to 0x5652ebdffba0 2026-03-24T02:34:24.318 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:24.312+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:34:24.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:24.312+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:34:24.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:24.312+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:34:24.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:24.316+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:34:25.312 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:25.305+0000 7fc159ee0640 -1 reset not still connected to 0x56417bee2f70 2026-03-24T02:34:25.320 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:25.313+0000 7fc159ee0640 -1 reset not still connected to 0x56417cdd40d0 2026-03-24T02:34:25.321 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:25.313+0000 7fc159ee0640 -1 reset not still connected to 0x56417d27a5b0 2026-03-24T02:34:25.321 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:25.313+0000 7fc159ee0640 -1 reset not still connected to 0x56417d2855f0 2026-03-24T02:34:25.321 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:25.313+0000 7fc159ee0640 -1 reset not still connected to 0x56417eb08d00 2026-03-24T02:34:25.321 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:25.313+0000 7fc159ee0640 -1 reset not still connected to 0x56417f9b6270 2026-03-24T02:34:25.321 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:25.313+0000 7fc159ee0640 -1 reset not still connected to 0x5641822a9c70 2026-03-24T02:34:25.321 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:25.313+0000 7fc159ee0640 -1 reset not still connected to 0x5641829e1d40 2026-03-24T02:34:25.321 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:25.313+0000 7fc159ee0640 -1 reset not still connected to 0x5641840be270 2026-03-24T02:34:27.129 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:27.124+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:34:27.129 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:27.124+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:34:27.129 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:27.124+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:34:27.129 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:27.124+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:34:27.129 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:27.124+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:34:27.210 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:27.204+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:34:27.216 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:27.212+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:34:27.462 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:27.456+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:34:29.544 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:29.533+0000 7fc159ee0640 -1 reset not still connected to 0x5641840be270 2026-03-24T02:34:30.932 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:30.928+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:34:30.932 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:30.928+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:34:30.932 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:30.928+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:34:30.932 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:30.928+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:34:30.983 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:30.980+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:34:30.984 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:30.980+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:34:30.984 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:30.980+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fa7ed800 finish: resending after timeout 2026-03-24T02:34:30.984 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:30.980+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:34:30.984 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:30.980+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:34:31.853 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:31.844+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:34:31.948 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:31.944+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:34:31.949 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:31.944+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda4f00 finish: resending after timeout 2026-03-24T02:34:31.949 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:31.944+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:34:31.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:31.944+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:34:32.134 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:32.128+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:34:32.135 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:32.128+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:34:32.138 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:32.132+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:34:32.139 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:32.132+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:34:32.139 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:32.132+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:34:32.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:32.208+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:34:32.216 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:32.212+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:34:32.461 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:32.456+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:34:35.934 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:35.928+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:34:35.934 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:35.928+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:34:35.934 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:35.928+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:34:35.934 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:35.928+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:34:35.983 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:35.980+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:34:35.984 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:35.980+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:34:35.984 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:35.980+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028240 finish: resending after timeout 2026-03-24T02:34:35.984 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:35.980+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:34:36.725 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:36.717+0000 7fc159ee0640 -1 reset not still connected to 0x56417bee2f70 2026-03-24T02:34:36.725 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:36.717+0000 7fc159ee0640 -1 reset not still connected to 0x56417cdd40d0 2026-03-24T02:34:36.727 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:36.721+0000 7fc159ee0640 -1 reset not still connected to 0x56417d2855f0 2026-03-24T02:34:36.730 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:36.725+0000 7fc159ee0640 -1 reset not still connected to 0x56417eb08d00 2026-03-24T02:34:36.738 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:36.729+0000 7fc159ee0640 -1 reset not still connected to 0x564180a31ee0 2026-03-24T02:34:36.738 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:36.729+0000 7fc159ee0640 -1 reset not still connected to 0x5641822a9c70 2026-03-24T02:34:36.738 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:36.729+0000 7fc159ee0640 -1 reset not still connected to 0x5641829e1d40 2026-03-24T02:34:36.738 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:36.729+0000 7fc159ee0640 -1 reset not still connected to 0x5641840be270 2026-03-24T02:34:36.738 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:36.729+0000 7fc159ee0640 -1 reset not still connected to 0x56419adcd860 2026-03-24T02:34:36.738 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:34:36.729+0000 7fc159ee0640 -1 reset not still connected to 0x5641a16cec30 2026-03-24T02:34:36.854 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:36.848+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:34:36.949 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:36.944+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:34:36.949 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:36.944+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:34:36.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:36.944+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:34:37.138 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:37.132+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:34:37.138 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:37.132+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:34:37.138 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:37.132+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:34:37.163 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:37.160+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:34:37.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:37.160+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:34:37.462 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:37.456+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:34:38.537 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:38.532+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:34:39.526 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:39.520+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:34:40.933 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:40.928+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:34:40.934 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:40.928+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:34:40.934 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:40.928+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:34:40.934 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:40.928+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:34:41.864 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:41.860+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:34:42.018 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:34:42.009+0000 7fb2796c9640 -1 reset not still connected to 0x55ac15a72270 2026-03-24T02:34:42.139 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:42.132+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:34:42.139 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:42.132+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9670000 finish: resending after timeout 2026-03-24T02:34:42.139 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:42.132+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:34:42.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:42.160+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:34:42.166 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:42.160+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:34:42.282 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:42.273+0000 7f5496ce4640 -1 reset not still connected to 0x5652c92eb520 2026-03-24T02:34:42.283 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:42.273+0000 7f5496ce4640 -1 reset not still connected to 0x5652c95dc8f0 2026-03-24T02:34:42.283 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:42.273+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c31380 2026-03-24T02:34:42.295 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:34:42.289+0000 7f5496ce4640 -1 reset not still connected to 0x5652ebdff2b0 2026-03-24T02:34:42.898 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:42.892+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:34:43.168 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:43.164+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:34:43.168 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:43.164+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:34:43.170 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:43.164+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:34:43.541 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:43.536+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:34:44.529 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:44.524+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:34:44.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-03-24T02:34:44.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep 'image health:' 2026-03-24T02:34:44.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut '-d ' -f 3 2026-03-24T02:34:45.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: failed to query services: (13) Permission denied 2026-03-24T02:34:45.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=WARNING 2026-03-24T02:34:45.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test WARNING = ERROR 2026-03-24T02:34:45.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test WARNING = OK 2026-03-24T02:34:45.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in `seq 1 40` 2026-03-24T02:34:45.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 4 -ne 1 2026-03-24T02:34:45.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 30 2026-03-24T02:34:45.320 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:45.316+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:34:45.320 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:45.316+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:34:45.320 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:45.316+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:34:47.139 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:47.136+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:34:47.142 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:47.136+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:34:47.166 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:47.160+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:34:47.168 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:47.164+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:34:47.286 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:47.280+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:34:47.290 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:47.284+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:34:47.290 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:47.284+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:34:47.290 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:47.284+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:34:47.914 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:47.908+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:34:48.170 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:48.164+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:34:48.170 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:48.164+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:34:48.170 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:48.164+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:34:48.543 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:48.540+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:34:49.530 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:49.524+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:34:50.321 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:50.316+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:34:50.321 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:50.316+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:34:50.321 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:50.316+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:34:51.243 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:51.236+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:34:52.140 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:52.136+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:34:52.141 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:52.136+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:34:52.167 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:52.160+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:34:52.168 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:52.164+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:34:52.287 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:52.284+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:34:52.292 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:52.288+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:34:52.294 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:52.288+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:34:52.294 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:34:52.288+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:34:52.915 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:52.908+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:34:53.546 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:53.540+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:34:53.744 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:53.740+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:34:53.745 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:53.740+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:34:53.745 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:34:53.740+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:34:54.530 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:54.524+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:34:56.243 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:56.240+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:34:57.142 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:57.136+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:34:57.142 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:57.136+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:34:57.168 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:57.164+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:34:57.169 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:57.164+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:34:57.242 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:57.236+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:34:57.243 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:57.236+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:34:57.243 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:34:57.236+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:34:57.916 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:57.912+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:34:58.549 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:58.544+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:34:59.535 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:34:59.532+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:35:01.126 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:01.117+0000 7fc159ee0640 -1 reset not still connected to 0x56419126a5b0 2026-03-24T02:35:02.243 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:35:02.240+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:35:02.243 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:35:02.240+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:35:02.243 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:35:02.240+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:35:02.897 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:35:02.892+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:35:02.897 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:35:02.892+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:35:02.897 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:35:02.892+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:35:02.917 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:02.912+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:35:03.548 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:03.544+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:35:06.298 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:35:06.292+0000 7f773bdb3640 -1 reset not still connected to 0x5647ae2d3450 2026-03-24T02:35:06.298 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:35:06.292+0000 7f773bdb3640 -1 reset not still connected to 0x5647ca6aa270 2026-03-24T02:35:06.299 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:35:06.296+0000 7f773bdb3640 -1 reset not still connected to 0x5647ce8f9c70 2026-03-24T02:35:06.355 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:06.352+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:35:06.355 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:06.352+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:35:06.355 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:06.352+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:35:06.355 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:06.352+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:35:06.355 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:06.352+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:35:06.356 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:06.352+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:35:07.245 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:35:07.240+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:35:07.245 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:35:07.240+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:35:07.245 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:35:07.240+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:35:07.432 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:07.428+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:35:07.433 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:07.428+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a000 finish: resending after timeout 2026-03-24T02:35:07.433 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:07.428+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:35:07.433 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:07.428+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:35:07.898 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:35:07.892+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:35:07.901 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:35:07.892+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:35:07.902 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:35:07.892+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:35:07.920 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:07.916+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:35:08.470 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:08.465+0000 7fc159ee0640 -1 reset not still connected to 0x56417cdd5860 2026-03-24T02:35:08.676 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:08.669+0000 7fc159ee0640 -1 reset not still connected to 0x56417cdd40d0 2026-03-24T02:35:08.689 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:08.681+0000 7fc159ee0640 -1 reset not still connected to 0x56417d27a5b0 2026-03-24T02:35:08.689 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:08.681+0000 7fc159ee0640 -1 reset not still connected to 0x56417d2855f0 2026-03-24T02:35:08.689 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:08.681+0000 7fc159ee0640 -1 reset not still connected to 0x56417eb08d00 2026-03-24T02:35:08.689 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:08.681+0000 7fc159ee0640 -1 reset not still connected to 0x56417f9b6270 2026-03-24T02:35:08.689 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:08.681+0000 7fc159ee0640 -1 reset not still connected to 0x56417f9b7ad0 2026-03-24T02:35:08.693 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:08.685+0000 7fc159ee0640 -1 reset not still connected to 0x564180a31ee0 2026-03-24T02:35:08.693 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:08.685+0000 7fc159ee0640 -1 reset not still connected to 0x5641822a9c70 2026-03-24T02:35:08.693 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:08.685+0000 7fc159ee0640 -1 reset not still connected to 0x5641829e1d40 2026-03-24T02:35:08.693 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:08.685+0000 7fc159ee0640 -1 reset not still connected to 0x5641840be270 2026-03-24T02:35:08.693 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:08.685+0000 7fc159ee0640 -1 reset not still connected to 0x56419adcd860 2026-03-24T02:35:08.693 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:08.685+0000 7fc159ee0640 -1 reset not still connected to 0x5641a16cec30 2026-03-24T02:35:11.234 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.3110509599020411) > weight (0.25) 2026-03-24T02:35:11.234 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.3239699236785497) > weight (0.25) 2026-03-24T02:35:11.234 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-03-24T02:35:11.234 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-03-24T02:35:11.234 INFO:tasks.rbd_mirror_thrash:waiting for 3.958849598818075 secs before reviving daemons 2026-03-24T02:35:11.235 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:35:11.232+0000 7fbe95a7e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 39638) UID: 1000 2026-03-24T02:35:12.245 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:35:12.240+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f134e53ec0 finish: resending after timeout 2026-03-24T02:35:12.245 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:35:12.240+0000 7fbe9527d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f1348e5680 finish: resending after timeout 2026-03-24T02:35:12.245 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:35:12.240+0000 7fbe94a7c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55f13eda5740 finish: resending after timeout 2026-03-24T02:35:12.435 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:12.428+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:35:12.435 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:12.428+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:35:12.435 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:12.432+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:35:13.335 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:13.328+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:35:13.335 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:13.328+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:35:14.593 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:14.588+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:35:15.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep 'image health:' 2026-03-24T02:35:15.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut '-d ' -f 3 2026-03-24T02:35:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-03-24T02:35:15.194 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-03-24T02:35:15.194 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:35:15.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: failed to query services: (13) Permission denied 2026-03-24T02:35:15.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=WARNING 2026-03-24T02:35:15.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test WARNING = ERROR 2026-03-24T02:35:15.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test WARNING = OK 2026-03-24T02:35:15.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in `seq 1 40` 2026-03-24T02:35:15.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 5 -ne 1 2026-03-24T02:35:15.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 30 2026-03-24T02:35:16.200 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:16.196+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:35:16.213 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:16.208+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:35:16.213 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:16.208+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2840 finish: resending after timeout 2026-03-24T02:35:16.213 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:16.208+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:35:16.214 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:16.208+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:35:17.525 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:35:17.520+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:35:17.525 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:35:17.520+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:35:17.525 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:35:17.520+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x563700d08480 finish: resending after timeout 2026-03-24T02:35:18.336 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:18.332+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:35:18.336 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:18.332+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:35:19.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:19.548+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:35:19.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:19.548+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:35:19.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:19.548+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4b5c0 finish: resending after timeout 2026-03-24T02:35:20.137 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:35:20.132+0000 7f773bdb3640 -1 reset not still connected to 0x5647ce8f9c70 2026-03-24T02:35:21.214 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:21.208+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:35:21.216 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:21.212+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:35:21.296 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-03-24T02:35:21.296 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-03-24T02:35:21.296 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-03-24T02:35:21.297 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-03-24T02:35:21.298 INFO:tasks.rbd_mirror_thrash:waiting for 80.27453669719986 secs before thrashing 2026-03-24T02:35:23.864 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:35:23.857+0000 7fb2796c9640 -1 reset not still connected to 0x55ac15c3d040 2026-03-24T02:35:23.864 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:35:23.857+0000 7fb2796c9640 -1 reset not still connected to 0x55ac1b06e4e0 2026-03-24T02:35:23.864 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:35:23.857+0000 7fb2796c9640 -1 reset not still connected to 0x55ac2d71b040 2026-03-24T02:35:23.871 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:35:23.865+0000 7fb2796c9640 -1 reset not still connected to 0x55ac32b66340 2026-03-24T02:35:23.932 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:23.928+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:35:24.556 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:24.552+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:35:24.556 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:24.552+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:35:24.954 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:24.948+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:35:24.954 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:24.948+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea7740 finish: resending after timeout 2026-03-24T02:35:25.147 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:25.140+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:35:25.153 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:25.148+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:35:25.185 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:35:25.177+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c5dad0 2026-03-24T02:35:26.260 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:35:26.256+0000 7f56058d6640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:35:26.261 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:35:26.256+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb028d80 finish: resending after timeout 2026-03-24T02:35:29.431 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:35:29.424+0000 7f773bdb3640 -1 reset not still connected to 0x5647b63815f0 2026-03-24T02:35:29.434 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:35:29.428+0000 7f773bdb3640 -1 reset not still connected to 0x5647bae92a90 2026-03-24T02:35:29.435 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:35:29.428+0000 7f773bdb3640 -1 reset not still connected to 0x5647c97d6680 2026-03-24T02:35:29.437 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:35:29.432+0000 7f773bdb3640 -1 reset not still connected to 0x5647d0992820 2026-03-24T02:35:29.557 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:29.552+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:35:29.557 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:29.552+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:35:30.187 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:30.180+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:35:30.189 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:30.184+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:35:30.189 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:30.184+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:35:32.639 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:32.632+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:35:32.639 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:32.636+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:35:33.825 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:33.820+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:35:37.496 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:37.492+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6c41bbd40 finish: resending after timeout 2026-03-24T02:35:37.640 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:37.636+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:35:37.640 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:37.636+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:35:38.825 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:38.820+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:35:38.842 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:38.836+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:35:38.848 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:38.844+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:35:39.071 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:35:39.064+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:35:40.067 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:40.064+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b7dd2fc0 finish: resending after timeout 2026-03-24T02:35:40.068 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:40.064+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:35:40.071 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:40.064+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:35:42.300 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:35:42.289+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c31ad0 2026-03-24T02:35:42.300 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:35:42.293+0000 7f5496ce4640 -1 reset not still connected to 0x5652d23d2f70 2026-03-24T02:35:42.300 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:35:42.293+0000 7f5496ce4640 -1 reset not still connected to 0x5652d5c3f5f0 2026-03-24T02:35:42.303 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:35:42.297+0000 7f5496ce4640 -1 reset not still connected to 0x5652ebdffba0 2026-03-24T02:35:42.497 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:42.492+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6c41bbd40 finish: resending after timeout 2026-03-24T02:35:43.826 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:43.820+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:35:43.843 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:43.840+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764d91b680 finish: resending after timeout 2026-03-24T02:35:43.849 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:43.844+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:35:44.900 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:44.896+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:35:44.900 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:44.896+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:35:45.074 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:45.068+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:35:45.075 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:45.068+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:35:45.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-03-24T02:35:45.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut '-d ' -f 3 2026-03-24T02:35:45.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep 'image health:' 2026-03-24T02:35:45.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: failed to query services: (13) Permission denied 2026-03-24T02:35:45.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=WARNING 2026-03-24T02:35:45.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test WARNING = ERROR 2026-03-24T02:35:45.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test WARNING = OK 2026-03-24T02:35:45.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in `seq 1 40` 2026-03-24T02:35:45.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 6 -ne 1 2026-03-24T02:35:45.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 30 2026-03-24T02:35:47.524 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.4395072335956671) > weight (0.25) 2026-03-24T02:35:47.525 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6560194776751185) > weight (0.25) 2026-03-24T02:35:47.525 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-03-24T02:35:47.525 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-03-24T02:35:47.525 INFO:tasks.rbd_mirror_thrash:waiting for 1.5448499862203235 secs before reviving daemons 2026-03-24T02:35:47.526 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:35:47.520+0000 7f6df1c0b640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 49352) UID: 1000 2026-03-24T02:35:48.856 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:48.852+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:35:49.070 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-03-24T02:35:49.071 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:35:49.071 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-03-24T02:35:49.071 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-03-24T02:35:49.071 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-03-24T02:35:49.073 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-03-24T02:35:49.073 INFO:tasks.rbd_mirror_thrash:waiting for 110.84173305951153 secs before thrashing 2026-03-24T02:35:49.502 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:49.496+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6c41bbd40 finish: resending after timeout 2026-03-24T02:35:49.658 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:35:49.652+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:35:49.909 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:49.904+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:35:49.910 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:49.904+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:35:50.077 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:50.072+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:35:50.080 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:50.072+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:35:50.283 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:35:50.273+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c31380 2026-03-24T02:35:50.293 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:35:50.285+0000 7f5496ce4640 -1 reset not still connected to 0x5652ebdff2b0 2026-03-24T02:35:51.523 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:51.520+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:35:54.659 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:35:54.656+0000 7f56050d5640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x5636fb029b00 finish: resending after timeout 2026-03-24T02:35:55.343 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:55.337+0000 7fc159ee0640 -1 reset not still connected to 0x56418291a820 2026-03-24T02:35:55.353 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:55.345+0000 7fc159ee0640 -1 reset not still connected to 0x564185cc5a00 2026-03-24T02:35:55.353 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:55.345+0000 7fc159ee0640 -1 reset not still connected to 0x5641885a7110 2026-03-24T02:35:55.353 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:55.345+0000 7fc159ee0640 -1 reset not still connected to 0x56419126a340 2026-03-24T02:35:55.353 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:55.345+0000 7fc159ee0640 -1 reset not still connected to 0x56419126a5b0 2026-03-24T02:35:55.354 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:55.345+0000 7fc159ee0640 -1 reset not still connected to 0x56419e1771e0 2026-03-24T02:35:55.354 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:55.345+0000 7fc159ee0640 -1 reset not still connected to 0x5641a16ce270 2026-03-24T02:35:55.573 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:55.568+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:35:56.159 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:56.152+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:35:56.168 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:35:56.164+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764dcae0c0 finish: resending after timeout 2026-03-24T02:35:57.627 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:57.621+0000 7fc159ee0640 -1 reset not still connected to 0x56417cdd5860 2026-03-24T02:35:57.627 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:35:57.624+0000 7ff7ec37e640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55e9e6a08e40 finish: resending after timeout 2026-03-24T02:35:57.641 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:57.633+0000 7fc159ee0640 -1 reset not still connected to 0x56417eb08410 2026-03-24T02:35:57.641 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:57.633+0000 7fc159ee0640 -1 reset not still connected to 0x5641822a91e0 2026-03-24T02:35:57.641 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:35:57.633+0000 7fc159ee0640 -1 reset not still connected to 0x56418eb168f0 2026-03-24T02:35:58.570 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:58.564+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b81a7080 finish: resending after timeout 2026-03-24T02:35:58.570 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:58.564+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6c41bbd40 finish: resending after timeout 2026-03-24T02:35:58.572 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:58.568+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:35:58.577 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:35:58.572+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b9169ec0 finish: resending after timeout 2026-03-24T02:36:00.574 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:00.568+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:36:01.165 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:36:01.160+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:36:01.170 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:36:01.164+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764dcae0c0 finish: resending after timeout 2026-03-24T02:36:02.630 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:36:02.624+0000 7ff7ebb7d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55e9e6a08e40 finish: resending after timeout 2026-03-24T02:36:03.034 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:03.028+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:36:03.573 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:03.568+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b81a7080 finish: resending after timeout 2026-03-24T02:36:03.573 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:03.568+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6c41bbd40 finish: resending after timeout 2026-03-24T02:36:03.808 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:03.804+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:36:06.168 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:36:06.164+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:36:06.174 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:36:06.168+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764dcae0c0 finish: resending after timeout 2026-03-24T02:36:06.864 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:36:06.860+0000 7f085e5d7640 -1 reset not still connected to 0x5587b0c24680 2026-03-24T02:36:06.869 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:36:06.864+0000 7f085e5d7640 -1 reset not still connected to 0x5587b0c25e10 2026-03-24T02:36:06.878 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:36:06.872+0000 7f085e5d7640 -1 reset not still connected to 0x5587b692ed00 2026-03-24T02:36:08.018 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:08.012+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:36:08.584 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:08.576+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b81a7080 finish: resending after timeout 2026-03-24T02:36:08.584 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:08.576+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6c41bbd40 finish: resending after timeout 2026-03-24T02:36:10.287 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:10.284+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:36:10.288 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:10.284+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:36:10.463 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:36:10.460+0000 7ff7ebb7d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55e9e6a08e40 finish: resending after timeout 2026-03-24T02:36:11.171 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:36:11.168+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764ce4a6c0 finish: resending after timeout 2026-03-24T02:36:11.178 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:36:11.172+0000 7f414ab10640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764dcae0c0 finish: resending after timeout 2026-03-24T02:36:12.761 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:36:12.753+0000 7fc159ee0640 -1 reset not still connected to 0x5641817871e0 2026-03-24T02:36:12.762 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:36:12.753+0000 7fc159ee0640 -1 reset not still connected to 0x5641829e0340 2026-03-24T02:36:12.762 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:36:12.753+0000 7fc159ee0640 -1 reset not still connected to 0x5641874aa410 2026-03-24T02:36:12.762 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:36:12.753+0000 7fc159ee0640 -1 reset not still connected to 0x564199040b60 2026-03-24T02:36:12.762 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:36:12.753+0000 7fc159ee0640 -1 reset not still connected to 0x564199041ee0 2026-03-24T02:36:12.762 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:36:12.753+0000 7fc159ee0640 -1 reset not still connected to 0x56419adccea0 2026-03-24T02:36:12.762 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:36:12.753+0000 7fc159ee0640 -1 reset not still connected to 0x56419adcd6c0 2026-03-24T02:36:12.762 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:36:12.753+0000 7fc159ee0640 -1 reset not still connected to 0x56419adcdee0 2026-03-24T02:36:12.764 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:36:12.757+0000 7fc159ee0640 -1 reset not still connected to 0x56419d60fd40 2026-03-24T02:36:13.026 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:13.020+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:36:13.583 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:13.576+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b81a7080 finish: resending after timeout 2026-03-24T02:36:13.583 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:13.576+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6c41bbd40 finish: resending after timeout 2026-03-24T02:36:13.778 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:36:13.773+0000 7fc159ee0640 -1 reset not still connected to 0x56418291a820 2026-03-24T02:36:13.784 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:36:13.777+0000 7fc159ee0640 -1 reset not still connected to 0x56419126a5b0 2026-03-24T02:36:14.695 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:36:14.688+0000 7f773bdb3640 -1 reset not still connected to 0x5647a17d9d40 2026-03-24T02:36:14.695 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:36:14.692+0000 7f773bdb3640 -1 reset not still connected to 0x5647ae2d3450 2026-03-24T02:36:14.695 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:36:14.692+0000 7f773bdb3640 -1 reset not still connected to 0x5647c0cbc750 2026-03-24T02:36:14.696 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:36:14.692+0000 7f773bdb3640 -1 reset not still connected to 0x5647c89d21a0 2026-03-24T02:36:14.696 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:36:14.692+0000 7f773bdb3640 -1 reset not still connected to 0x5647ca6aa270 2026-03-24T02:36:14.696 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:36:14.692+0000 7f773bdb3640 -1 reset not still connected to 0x5647ce8f9c70 2026-03-24T02:36:14.696 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:36:14.692+0000 7f773bdb3640 -1 reset not still connected to 0x5647d0992410 2026-03-24T02:36:15.021 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:15.016+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c9980 finish: resending after timeout 2026-03-24T02:36:15.288 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:15.284+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:36:15.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut '-d ' -f 3 2026-03-24T02:36:15.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-03-24T02:36:15.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep 'image health:' 2026-03-24T02:36:15.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: failed to query services: (13) Permission denied 2026-03-24T02:36:15.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=WARNING 2026-03-24T02:36:15.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test WARNING = ERROR 2026-03-24T02:36:15.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test WARNING = OK 2026-03-24T02:36:15.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in `seq 1 40` 2026-03-24T02:36:15.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 7 -ne 1 2026-03-24T02:36:15.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 30 2026-03-24T02:36:15.464 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:36:15.460+0000 7ff7ebb7d640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55e9e6a08e40 finish: resending after timeout 2026-03-24T02:36:15.896 INFO:tasks.ceph.cluster2.osd.0.vm07.stderr:2026-03-24T02:36:15.892+0000 7f773bdb3640 -1 reset not still connected to 0x5647bae93e10 2026-03-24T02:36:16.188 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:36:16.184+0000 7f414a30f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55764dcae0c0 finish: resending after timeout 2026-03-24T02:36:16.850 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:36:16.841+0000 7fc159ee0640 -1 reset not still connected to 0x56418eb168f0 2026-03-24T02:36:16.850 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:36:16.841+0000 7fc159ee0640 -1 reset not still connected to 0x56419126b110 2026-03-24T02:36:16.850 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:36:16.841+0000 7fc159ee0640 -1 reset not still connected to 0x56419adcde10 2026-03-24T02:36:16.850 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:36:16.841+0000 7fc159ee0640 -1 reset not still connected to 0x56419e177040 2026-03-24T02:36:16.850 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:36:16.841+0000 7fc159ee0640 -1 reset not still connected to 0x5641a5e4f520 2026-03-24T02:36:18.594 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:18.585+0000 7f5496ce4640 -1 reset not still connected to 0x5652c68aaa90 2026-03-24T02:36:18.598 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:18.589+0000 7f5496ce4640 -1 reset not still connected to 0x5652c68abee0 2026-03-24T02:36:18.602 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:18.597+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f9380 2026-03-24T02:36:18.610 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:18.605+0000 7f5496ce4640 -1 reset not still connected to 0x5652c8147860 2026-03-24T02:36:18.610 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:18.605+0000 7f5496ce4640 -1 reset not still connected to 0x5652c83ca5b0 2026-03-24T02:36:18.610 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:18.605+0000 7f5496ce4640 -1 reset not still connected to 0x5652c83cbd40 2026-03-24T02:36:18.610 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:18.605+0000 7f5496ce4640 -1 reset not still connected to 0x5652c92eb520 2026-03-24T02:36:18.610 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:18.605+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c31380 2026-03-24T02:36:18.610 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:18.605+0000 7f5496ce4640 -1 reset not still connected to 0x5652d23d2d00 2026-03-24T02:36:18.610 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:18.605+0000 7f5496ce4640 -1 reset not still connected to 0x5652d23d31e0 2026-03-24T02:36:18.610 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:18.605+0000 7f5496ce4640 -1 reset not still connected to 0x5652e4d70000 2026-03-24T02:36:18.610 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:18.605+0000 7f5496ce4640 -1 reset not still connected to 0x5652ebdff2b0 2026-03-24T02:36:20.213 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:20.205+0000 7f5496ce4640 -1 reset not still connected to 0x5652c5367ba0 2026-03-24T02:36:20.214 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:20.205+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f8750 2026-03-24T02:36:20.229 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:20.221+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f9110 2026-03-24T02:36:20.229 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:20.221+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f91e0 2026-03-24T02:36:20.229 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:20.221+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f92b0 2026-03-24T02:36:20.229 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:20.221+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f9d40 2026-03-24T02:36:20.236 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:20.229+0000 7f5496ce4640 -1 reset not still connected to 0x5652c83cb930 2026-03-24T02:36:20.236 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:20.229+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c5dad0 2026-03-24T02:36:20.236 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:20.229+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1e2bc70 2026-03-24T02:36:20.236 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:20.229+0000 7f5496ce4640 -1 reset not still connected to 0x5652e4d71ad0 2026-03-24T02:36:20.236 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:20.229+0000 7f5496ce4640 -1 reset not still connected to 0x5652ebdfef70 2026-03-24T02:36:20.238 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:20.233+0000 7f5496ce4640 -1 reset not still connected to 0x5652ef237c70 2026-03-24T02:36:21.854 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:21.848+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea69c0 finish: resending after timeout 2026-03-24T02:36:22.181 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:22.176+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:36:22.249 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:22.244+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a24300 finish: resending after timeout 2026-03-24T02:36:26.280 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:36:26.273+0000 7fb2796c9640 -1 reset not still connected to 0x55ac138afe10 2026-03-24T02:36:26.280 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:36:26.273+0000 7fb2796c9640 -1 reset not still connected to 0x55ac14393a00 2026-03-24T02:36:27.181 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:27.176+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:36:27.277 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:27.272+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a24300 finish: resending after timeout 2026-03-24T02:36:30.972 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:30.968+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6c41bbd40 finish: resending after timeout 2026-03-24T02:36:30.972 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:30.968+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:36:32.198 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:32.192+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b8ea6480 finish: resending after timeout 2026-03-24T02:36:32.283 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:32.276+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a24300 finish: resending after timeout 2026-03-24T02:36:33.066 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:33.057+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f9380 2026-03-24T02:36:34.758 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:36:34.749+0000 7fb2796c9640 -1 reset not still connected to 0x55ac2d71b040 2026-03-24T02:36:34.758 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:36:34.749+0000 7fb2796c9640 -1 reset not still connected to 0x55ac312ec820 2026-03-24T02:36:34.758 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:36:34.749+0000 7fb2796c9640 -1 reset not still connected to 0x55ac32b66340 2026-03-24T02:36:35.472 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:35.465+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f9110 2026-03-24T02:36:35.472 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:35.465+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f91e0 2026-03-24T02:36:35.472 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:35.465+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f92b0 2026-03-24T02:36:35.472 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:35.465+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f9d40 2026-03-24T02:36:35.472 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:35.465+0000 7f5496ce4640 -1 reset not still connected to 0x5652c83cb930 2026-03-24T02:36:35.472 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:35.465+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c5dad0 2026-03-24T02:36:35.474 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:35.465+0000 7f5496ce4640 -1 reset not still connected to 0x5652ebdfef70 2026-03-24T02:36:35.484 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:35.473+0000 7f5496ce4640 -1 reset not still connected to 0x5652ef237c70 2026-03-24T02:36:37.284 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:37.280+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a24300 finish: resending after timeout 2026-03-24T02:36:40.487 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:40.484+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:36:41.573 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-03-24T02:36:41.573 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-03-24T02:36:41.573 INFO:tasks.rbd_mirror_thrash:waiting for 7.709454847260595 secs before reviving daemons 2026-03-24T02:36:41.574 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:41.568+0000 7f9f33d71640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 27580) UID: 1000 2026-03-24T02:36:41.640 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:41.636+0000 7f9f2cd63640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55c6b96a3500 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-03-24T02:36:41.640 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:41.636+0000 7f9f2cd63640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55c6b96a3500 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-03-24T02:36:45.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-03-24T02:36:45.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut '-d ' -f 3 2026-03-24T02:36:45.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep 'image health:' 2026-03-24T02:36:45.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: failed to query services: (13) Permission denied 2026-03-24T02:36:45.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=WARNING 2026-03-24T02:36:45.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test WARNING = ERROR 2026-03-24T02:36:45.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test WARNING = OK 2026-03-24T02:36:45.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in `seq 1 40` 2026-03-24T02:36:45.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 8 -ne 1 2026-03-24T02:36:45.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 30 2026-03-24T02:36:45.489 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:45.484+0000 7f9f33570640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b31c8900 finish: resending after timeout 2026-03-24T02:36:46.505 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:36:46.497+0000 7fc159ee0640 -1 reset not still connected to 0x56419126a5b0 2026-03-24T02:36:46.705 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:46.700+0000 7f9f32d6f640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55c6b2a24300 finish: resending after timeout 2026-03-24T02:36:48.367 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:36:48.364+0000 7f55ff0c9640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x5636fb128000 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-03-24T02:36:48.367 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:36:48.364+0000 7f55ff0c9640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x5636fb128000 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-03-24T02:36:49.284 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-03-24T02:36:49.284 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:36:49.284 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-03-24T02:36:49.284 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-03-24T02:36:49.284 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-03-24T02:36:49.285 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-03-24T02:36:49.285 INFO:tasks.rbd_mirror_thrash:waiting for 91.26030584957374 secs before thrashing 2026-03-24T02:36:52.718 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:36:52.709+0000 7fb2796c9640 -1 reset not still connected to 0x55ac15c3dad0 2026-03-24T02:36:52.721 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:36:52.713+0000 7fb2796c9640 -1 reset not still connected to 0x55ac19f78c30 2026-03-24T02:36:52.721 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:36:52.713+0000 7fb2796c9640 -1 reset not still connected to 0x55ac2004f5f0 2026-03-24T02:36:52.730 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:36:52.721+0000 7fb2796c9640 -1 reset not still connected to 0x55ac2004fba0 2026-03-24T02:36:52.730 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:36:52.721+0000 7fb2796c9640 -1 reset not still connected to 0x55ac2d71a8f0 2026-03-24T02:36:52.730 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:36:52.721+0000 7fb2796c9640 -1 reset not still connected to 0x55ac32b66a90 2026-03-24T02:36:52.867 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:36:52.861+0000 7fb2796c9640 -1 reset not still connected to 0x55ac32b66a90 2026-03-24T02:36:53.469 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:36:53.464+0000 7f55ff0c9640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x5636fa6f2700 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-03-24T02:36:53.469 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:36:53.464+0000 7f55ff0c9640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x5636fa6f2700 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-03-24T02:36:56.116 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:56.112+0000 7f4287301640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55b9245a8e00 get_replay_status: replay not running 2026-03-24T02:36:56.130 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:36:56.124+0000 7f4287301640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55b923f9c000 get_replay_status: replay not running 2026-03-24T02:36:56.589 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:56.581+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c31ad0 2026-03-24T02:36:56.596 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:56.589+0000 7f5496ce4640 -1 reset not still connected to 0x5652d5a3c820 2026-03-24T02:36:56.596 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:56.589+0000 7f5496ce4640 -1 reset not still connected to 0x5652d5c3f5f0 2026-03-24T02:36:56.596 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:56.589+0000 7f5496ce4640 -1 reset not still connected to 0x5652d5c3fa00 2026-03-24T02:36:56.596 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:56.589+0000 7f5496ce4640 -1 reset not still connected to 0x5652e4d71110 2026-03-24T02:36:56.596 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:56.589+0000 7f5496ce4640 -1 reset not still connected to 0x5652ec2ac1a0 2026-03-24T02:36:56.596 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:36:56.589+0000 7f5496ce4640 -1 reset not still connected to 0x5652ef237c70 2026-03-24T02:36:59.043 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:36:59.037+0000 7fb2796c9640 -1 reset not still connected to 0x55ac32b66a90 2026-03-24T02:37:09.413 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:37:09.405+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1e2ab60 2026-03-24T02:37:09.416 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:37:09.409+0000 7f5496ce4640 -1 reset not still connected to 0x5652ebdff2b0 2026-03-24T02:37:15.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-03-24T02:37:15.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep 'image health:' 2026-03-24T02:37:15.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut '-d ' -f 3 2026-03-24T02:37:15.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: failed to query services: (13) Permission denied 2026-03-24T02:37:15.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=OK 2026-03-24T02:37:15.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test OK = ERROR 2026-03-24T02:37:15.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test OK = OK 2026-03-24T02:37:15.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:15.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_pool_images cluster1 mirror 50 2026-03-24T02:37:15.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:15.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:15.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_count=50 2026-03-24T02:37:15.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:37:15.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local count 2026-03-24T02:37:15.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_count=0 2026-03-24T02:37:15.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:37:15.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 1 40 2026-03-24T02:37:15.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in `seq 1 40` 2026-03-24T02:37:15.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 1 -ne 1 2026-03-24T02:37:15.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-03-24T02:37:15.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep 'images: ' 2026-03-24T02:37:15.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut '-d ' -f 2 2026-03-24T02:37:15.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: failed to query services: (13) Permission denied 2026-03-24T02:37:15.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ count=50 2026-03-24T02:37:15.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 50 = 50 2026-03-24T02:37:15.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:15.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_pool_healthy cluster1 mirror 2026-03-24T02:37:15.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:15.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:15.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:37:15.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state 2026-03-24T02:37:15.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 1 40 2026-03-24T02:37:15.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in `seq 1 40` 2026-03-24T02:37:15.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 1 -ne 1 2026-03-24T02:37:15.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-03-24T02:37:15.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep 'image health:' 2026-03-24T02:37:15.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut '-d ' -f 3 2026-03-24T02:37:15.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: failed to query services: (13) Permission denied 2026-03-24T02:37:15.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=OK 2026-03-24T02:37:15.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test OK = ERROR 2026-03-24T02:37:15.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test OK = OK 2026-03-24T02:37:15.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:15.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ testlog 'TEST: compare many images' 2026-03-24T02:37:15.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ date '+%F %T' 2026-03-24T02:37:15.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ tee -a /tmp/tmp.l3GEA596QR/rbd-mirror.test.log 2026-03-24T02:37:15.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo 2026-03-24 02:37:15 TEST: compare many images 2026-03-24T02:37:15.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:2026-03-24 02:37:15 TEST: compare many images 2026-03-24T02:37:15.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 1 50 2026-03-24T02:37:15.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:37:15.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_1 2026-03-24T02:37:15.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_1 snap 2026-03-24T02:37:15.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:15.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:15.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:37:15.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:37:15.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_1@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:37:16.383 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:37:16.377+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1e2ab60 2026-03-24T02:37:16.383 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:37:16.377+0000 7f5496ce4640 -1 reset not still connected to 0x5652ebdff2b0 2026-03-24T02:37:16.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_1 2026-03-24T02:37:16.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:16.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:16.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:37:16.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_1 started 2026-03-24T02:37:16.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:16.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:16.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:37:16.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:37:16.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:37:16.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:16.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:37:17.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_1 started 2026-03-24T02:37:17.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:17.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:17.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:37:17.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:37:17.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:37:17.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:37:17.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_1 2026-03-24T02:37:17.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:37:17.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:37:17.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:17.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:17.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:37:17.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:37:17.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:17.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:37:17.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:17.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:37:17.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_1 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:37:17.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:17.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:37:17.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:37:17.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:37:17.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:17.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:17.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:17.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:37:17.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:37:17.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:37:17.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:37:17.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:17.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:17.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:17.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:17.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_1 2026-03-24T02:37:17.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:37:17.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:37:17.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:37:17.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:37:17.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:37:17.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:37:17.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:37:17.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:17.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_1 2026-03-24T02:37:17.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:37:17.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:17.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:37:17.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:37:17.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:37:17.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:37:17.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_1 2026-03-24T02:37:17.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:37:17.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:17.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:37:17.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:37:17.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:37:17.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:37:17.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:37:17.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:37:17.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:17.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:37:18.132 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:37:18.125+0000 7f5496ce4640 -1 reset not still connected to 0x5652c92ea5b0 2026-03-24T02:37:18.132 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:37:18.125+0000 7f5496ce4640 -1 reset not still connected to 0x5652d714f930 2026-03-24T02:37:18.140 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:37:18.133+0000 7f5496ce4640 -1 reset not still connected to 0x5652ef236ea0 2026-03-24T02:37:18.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_1 2026-03-24T02:37:18.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:18.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:18.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:37:18.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:18.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_1 ']' 2026-03-24T02:37:18.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-03-24T02:37:18.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-03-24T02:37:18.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:18.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:18.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:18.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:18.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:18.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:18.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:18.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:18.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:18.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:18.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:18.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:18.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:18.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:18.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:18.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:18.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:18.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:18.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:18.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:18.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:18.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-03-24T02:37:18.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:18.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-03-24T02:37:18.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:18.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:18.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:18.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-03-24T02:37:18.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:18.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:18.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:18.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:18.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-03-24T02:37:18.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_1.status 2026-03-24T02:37:18.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:18.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:18.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-03-24T02:37:18.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:18.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 3 2026-03-24T02:37:18.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 3 2026-03-24T02:37:18.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:18.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:18.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=13, tag_tid=1, entry_tid=13601], [object_number=12, tag_tid=1, entry_tid=13600], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:18.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:18.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=13, tag_tid=2, entry_tid=1' 2026-03-24T02:37:18.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-03-24T02:37:18.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:18.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:18.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:18.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-03-24T02:37:18.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:18.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:18.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:18.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:18.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-03-24T02:37:18.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_1.status 2026-03-24T02:37:18.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:18.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:18.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-03-24T02:37:18.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:18.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 3 2026-03-24T02:37:18.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 3 2026-03-24T02:37:18.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:18.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:18.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=13, tag_tid=1, entry_tid=13601], [object_number=12, tag_tid=1, entry_tid=13600], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:18.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:18.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=13, tag_tid=1, entry_tid=13601' 2026-03-24T02:37:18.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=13, tag_tid=2, entry_tid=1' -a 'object_number=13, tag_tid=2, entry_tid=1' = 'object_number=13, tag_tid=1, entry_tid=13601' 2026-03-24T02:37:18.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=13, tag_tid=1, entry_tid=13601' '!=' '' 2026-03-24T02:37:18.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:37:18.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=13, tag_tid=1, entry_tid=13601' = '' 2026-03-24T02:37:18.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=13, tag_tid=1, entry_tid=13601' 2026-03-24T02:37:18.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:37:18.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:18.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=1' 2026-03-24T02:37:18.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=2 2026-03-24T02:37:18.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=13, tag_tid=1, entry_tid=13601' 2026-03-24T02:37:18.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:18.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:37:18.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=1 2026-03-24T02:37:18.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:37:18.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:18.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=1' 2026-03-24T02:37:18.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:37:18.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=13, tag_tid=1, entry_tid=13601' 2026-03-24T02:37:18.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:18.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:37:18.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=13601 2026-03-24T02:37:18.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 2 = 1 -a 1 -le 13601 2026-03-24T02:37:18.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:37:18.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:18.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:37:18.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_1 2026-03-24T02:37:18.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:18.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:18.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:37:18.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:18.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_1 ']' 2026-03-24T02:37:18.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-03-24T02:37:18.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-03-24T02:37:18.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:18.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:18.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:18.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:18.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:18.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:18.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:18.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:18.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:18.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:18.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-03-24T02:37:18.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:18.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:18.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:18.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:18.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:18.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:18.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:18.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:18.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:18.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:18.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:18.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:18.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:18.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:18.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:18.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-03-24T02:37:18.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:18.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-03-24T02:37:18.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:18.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:18.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:18.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-03-24T02:37:18.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:18.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:18.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:18.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:18.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-03-24T02:37:18.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_1.status 2026-03-24T02:37:18.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:18.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:18.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:18.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-03-24T02:37:18.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 3 2026-03-24T02:37:18.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 3 2026-03-24T02:37:18.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:18.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:18.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=13, tag_tid=1, entry_tid=13601], [object_number=12, tag_tid=1, entry_tid=13600], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:18.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:18.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=13, tag_tid=2, entry_tid=1' 2026-03-24T02:37:18.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-03-24T02:37:18.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:18.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:18.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:18.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-03-24T02:37:18.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:18.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:18.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:18.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:18.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-03-24T02:37:18.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_1.status 2026-03-24T02:37:18.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:18.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:18.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-03-24T02:37:18.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:18.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 3 2026-03-24T02:37:18.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 3 2026-03-24T02:37:18.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:18.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:18.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=13, tag_tid=1, entry_tid=13601], [object_number=12, tag_tid=1, entry_tid=13600], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:18.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:18.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=13, tag_tid=1, entry_tid=13601' 2026-03-24T02:37:18.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=13, tag_tid=2, entry_tid=1' -a 'object_number=13, tag_tid=2, entry_tid=1' = 'object_number=13, tag_tid=1, entry_tid=13601' 2026-03-24T02:37:18.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=13, tag_tid=1, entry_tid=13601' '!=' 'object_number=13, tag_tid=1, entry_tid=13601' 2026-03-24T02:37:18.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:18.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:37:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_1 2026-03-24T02:37:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:37:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_1 ']' 2026-03-24T02:37:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-03-24T02:37:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-03-24T02:37:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:19.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:19.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-03-24T02:37:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:19.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:19.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:19.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:19.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-03-24T02:37:19.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:19.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-03-24T02:37:19.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:19.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:19.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:19.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-03-24T02:37:19.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:19.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:19.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:19.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:19.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-03-24T02:37:19.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_1.status 2026-03-24T02:37:19.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:19.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:19.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-03-24T02:37:19.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:19.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 3 2026-03-24T02:37:19.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 3 2026-03-24T02:37:19.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:19.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:19.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=13, tag_tid=1, entry_tid=13601], [object_number=12, tag_tid=1, entry_tid=13600], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:19.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:19.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=13, tag_tid=2, entry_tid=1' 2026-03-24T02:37:19.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-03-24T02:37:19.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:19.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:19.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:19.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-03-24T02:37:19.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:19.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:19.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:19.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:19.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-03-24T02:37:19.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_1.status 2026-03-24T02:37:19.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:19.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:19.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-03-24T02:37:19.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:19.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 3 2026-03-24T02:37:19.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 3 2026-03-24T02:37:19.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:19.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:19.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=13, tag_tid=1, entry_tid=13601], [object_number=12, tag_tid=1, entry_tid=13600], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:19.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:19.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=13, tag_tid=1, entry_tid=13601' 2026-03-24T02:37:19.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=13, tag_tid=2, entry_tid=1' -a 'object_number=13, tag_tid=2, entry_tid=1' = 'object_number=13, tag_tid=1, entry_tid=13601' 2026-03-24T02:37:19.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=13, tag_tid=1, entry_tid=13601' '!=' 'object_number=13, tag_tid=1, entry_tid=13601' 2026-03-24T02:37:19.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:19.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:37:20.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_1 2026-03-24T02:37:20.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:20.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:20.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:37:20.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:20.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_1 ']' 2026-03-24T02:37:20.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-03-24T02:37:20.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-03-24T02:37:20.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:20.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:20.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:20.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:20.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:20.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:20.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:20.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:20.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:20.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:20.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-03-24T02:37:20.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:20.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:20.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:20.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:20.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:20.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:20.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:20.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:20.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:20.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:20.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:20.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:20.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:20.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:20.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:20.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:20.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:20.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:20.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:20.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:20.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:20.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:20.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:20.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:20.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:20.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:20.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:20.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:20.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:20.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:20.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:20.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:20.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:20.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-03-24T02:37:20.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:20.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-03-24T02:37:20.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:20.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:20.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:20.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-03-24T02:37:20.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:20.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:20.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:20.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:20.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-03-24T02:37:20.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_1.status 2026-03-24T02:37:20.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:20.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:20.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-03-24T02:37:20.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:20.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 3 2026-03-24T02:37:20.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 3 2026-03-24T02:37:20.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:20.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:20.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=13, tag_tid=1, entry_tid=13601], [object_number=12, tag_tid=1, entry_tid=13600], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:20.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:20.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=13, tag_tid=2, entry_tid=1' 2026-03-24T02:37:20.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-03-24T02:37:20.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:20.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:20.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:20.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-03-24T02:37:20.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:20.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:20.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:20.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:20.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-03-24T02:37:20.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_1.status 2026-03-24T02:37:20.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:20.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:20.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-03-24T02:37:20.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:20.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 3 2026-03-24T02:37:20.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 3 2026-03-24T02:37:20.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:20.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:20.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=13, tag_tid=1, entry_tid=13601], [object_number=12, tag_tid=1, entry_tid=13600], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:20.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:20.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=13, tag_tid=1, entry_tid=13601' 2026-03-24T02:37:20.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=13, tag_tid=2, entry_tid=1' -a 'object_number=13, tag_tid=2, entry_tid=1' = 'object_number=13, tag_tid=1, entry_tid=13601' 2026-03-24T02:37:20.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=13, tag_tid=1, entry_tid=13601' '!=' 'object_number=13, tag_tid=1, entry_tid=13601' 2026-03-24T02:37:20.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:20.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:37:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_1 2026-03-24T02:37:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:37:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_1 ']' 2026-03-24T02:37:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-03-24T02:37:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-03-24T02:37:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:22.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:22.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-03-24T02:37:22.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:22.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:22.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:22.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:22.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:22.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:22.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:22.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:22.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:22.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:22.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:22.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:22.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:22.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:22.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:22.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:22.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:22.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:22.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:22.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:22.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:22.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:22.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:22.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:22.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:22.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:22.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:22.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:22.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:22.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:22.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:22.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:22.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:22.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-03-24T02:37:22.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:22.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-03-24T02:37:22.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:22.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:22.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:22.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-03-24T02:37:22.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:22.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:22.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:22.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:22.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-03-24T02:37:22.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_1.status 2026-03-24T02:37:22.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:22.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:22.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-03-24T02:37:22.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:22.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 3 2026-03-24T02:37:22.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 3 2026-03-24T02:37:22.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:22.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:22.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:22.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:22.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=13, tag_tid=2, entry_tid=1' 2026-03-24T02:37:22.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-03-24T02:37:22.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:22.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:22.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:22.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-03-24T02:37:22.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:22.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:22.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:37:22.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:22.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-03-24T02:37:22.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_1.status 2026-03-24T02:37:22.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:22.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:22.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-03-24T02:37:22.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:22.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 3 2026-03-24T02:37:22.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 3 2026-03-24T02:37:22.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:22.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=13599], [object_number=14, tag_tid=1, entry_tid=13598]]], state=connected] 2026-03-24T02:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.status 2026-03-24T02:37:22.390 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=13, tag_tid=2, entry_tid=1' 2026-03-24T02:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=13, tag_tid=2, entry_tid=1' -a 'object_number=13, tag_tid=2, entry_tid=1' = 'object_number=13, tag_tid=2, entry_tid=1' 2026-03-24T02:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_1 snap 2026-03-24T02:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:37:23.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_1@snap 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_1': 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10d15a627331 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10d15a627331 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:25:41 2026 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:25:41 2026 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:26:42 2026 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10d15a627331 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: a574f0a7-a093-4469-b1ff-bb5da412c56c 2026-03-24T02:37:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:37:23.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:23.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_1 snap 2026-03-24T02:37:23.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:23.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:37:23.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:37:23.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:37:23.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.export 2026-03-24T02:37:23.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_1.export 2026-03-24T02:37:23.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_1.export 2026-03-24T02:37:23.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_1@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.export 2026-03-24T02:37:23.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:37:23.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_1@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_1.export 2026-03-24T02:37:24.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:37:24.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_1.export 2026-03-24T02:37:24.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_1.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_1.export 2026-03-24T02:37:24.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:24.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:37:24.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_2 2026-03-24T02:37:24.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_2 snap 2026-03-24T02:37:24.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:24.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:24.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:37:24.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:37:24.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_2@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:37:24.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_2 2026-03-24T02:37:24.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:24.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:24.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:37:24.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_2 started 2026-03-24T02:37:24.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:24.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:24.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:37:24.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:37:24.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:37:24.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:24.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:37:25.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_2 started 2026-03-24T02:37:25.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:25.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:25.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:37:25.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:37:25.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:37:25.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:37:25.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_2 2026-03-24T02:37:25.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:37:25.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:37:25.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:25.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:25.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:37:25.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:37:25.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:25.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:37:25.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:25.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:37:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_2 2026-03-24T02:37:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:37:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:37:25.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:37:25.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_2 2026-03-24T02:37:25.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 1 closest matches: 2026-03-24T02:37:25.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status mirror/image_22 2026-03-24T02:37:25.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:25.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:25.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_2 2026-03-24T02:37:25.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:37:25.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:25.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:25.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:25.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:25.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_2 2026-03-24T02:37:26.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 5 closest matches: 2026-03-24T02:37:26.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status mirror/image_21 2026-03-24T02:37:26.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status mirror/image_24 2026-03-24T02:37:26.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status mirror/image_25 2026-03-24T02:37:26.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status mirror/image_27 2026-03-24T02:37:26.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status mirror/image_28 2026-03-24T02:37:26.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:26.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:26.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_2 2026-03-24T02:37:26.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:37:26.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:26.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:26.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:26.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:26.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:37:26.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:26.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:26.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:26.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:37:26.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:26.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:26.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:26.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:26.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_2 2026-03-24T02:37:26.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 3 closest matches: 2026-03-24T02:37:26.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status mirror/image_23 2026-03-24T02:37:26.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status mirror/image_26 2026-03-24T02:37:26.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status mirror/image_29 2026-03-24T02:37:26.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_2 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:37:26.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:37:26.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:37:26.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:37:26.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:26.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:37:26.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:26.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:37:26.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:37:26.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:37:26.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:37:26.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:37:26.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:26.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:26.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:26.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_2 2026-03-24T02:37:26.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:37:26.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:37:26.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:37:26.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:37:26.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:37:26.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:37:26.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:37:26.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_2 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_2 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:26.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:37:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_2 2026-03-24T02:37:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:37:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_2 ']' 2026-03-24T02:37:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-03-24T02:37:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-03-24T02:37:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:26.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:26.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:26.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:26.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:26.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:26.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-03-24T02:37:26.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 1 closest matches: 2026-03-24T02:37:26.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_22 2026-03-24T02:37:26.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:26.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:26.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-03-24T02:37:26.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:37:26.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:26.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:26.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:26.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:37:26.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:26.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:26.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:26.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:37:26.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:26.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:26.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:26.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:37:26.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:26.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:26.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:37:26.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:26.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:26.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:26.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:26.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:26.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:26.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:26.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:26.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:26.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:26.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:26.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:26.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:26.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:26.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:26.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:26.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:26.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-03-24T02:37:26.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 5 closest matches: 2026-03-24T02:37:26.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_21 2026-03-24T02:37:26.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_24 2026-03-24T02:37:26.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_25 2026-03-24T02:37:26.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_27 2026-03-24T02:37:26.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_28 2026-03-24T02:37:26.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:26.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:26.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-03-24T02:37:26.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:37:26.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:26.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:26.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:26.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:37:26.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:26.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:26.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:26.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:37:26.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:26.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:26.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:26.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:37:26.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:26.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:26.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:37:26.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:26.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:26.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:26.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:26.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:26.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:26.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:26.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:26.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:26.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:26.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:26.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:26.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:26.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:26.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:26.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:26.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:26.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-03-24T02:37:26.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 3 closest matches: 2026-03-24T02:37:26.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_23 2026-03-24T02:37:26.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_26 2026-03-24T02:37:26.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_29 2026-03-24T02:37:26.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:26.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:26.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_2 2026-03-24T02:37:26.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:37:26.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:26.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:26.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:37:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:37:26.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:26.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:26.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:26.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_2 2026-03-24T02:37:26.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:26.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-03-24T02:37:26.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:26.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:26.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:37:26.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-03-24T02:37:26.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:26.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:26.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:37:26.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:26.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-03-24T02:37:26.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_2.status 2026-03-24T02:37:26.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:26.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:26.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-03-24T02:37:26.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:26.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:26.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:26.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:26.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6087], [object_number=6, tag_tid=1, entry_tid=6086]]], state=connected] 2026-03-24T02:37:26.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6087], [object_number=6, tag_tid=1, entry_tid=6086], [object_number=5, tag_tid=1, entry_tid=6085], [object_number=4, tag_tid=1, entry_tid=6084]]], state=connected] 2026-03-24T02:37:26.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:26.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:26.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-03-24T02:37:26.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:26.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:26.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:37:26.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-03-24T02:37:26.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:26.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:26.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:37:26.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:26.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-03-24T02:37:26.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_2.status 2026-03-24T02:37:26.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:26.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:26.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-03-24T02:37:26.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:26.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:26.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:26.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:26.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6087], [object_number=6, tag_tid=1, entry_tid=6086]]], state=connected] 2026-03-24T02:37:26.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6087], [object_number=6, tag_tid=1, entry_tid=6086], [object_number=5, tag_tid=1, entry_tid=6085], [object_number=4, tag_tid=1, entry_tid=6084]]], state=connected] 2026-03-24T02:37:26.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=6087' 2026-03-24T02:37:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=6087' 2026-03-24T02:37:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6087' '!=' '' 2026-03-24T02:37:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:37:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6087' = '' 2026-03-24T02:37:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=1, entry_tid=6087' 2026-03-24T02:37:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:37:26.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:37:26.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=1, entry_tid=6087' 2026-03-24T02:37:26.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:37:26.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:26.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=1 2026-03-24T02:37:26.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:26.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:37:26.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:26.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:37:26.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=1, entry_tid=6087' 2026-03-24T02:37:26.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:37:26.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:26.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=6087 2026-03-24T02:37:26.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 1 -a 1 -le 6087 2026-03-24T02:37:26.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:37:26.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:26.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:37:27.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_2 2026-03-24T02:37:27.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:27.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:27.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:37:27.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:27.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_2 ']' 2026-03-24T02:37:27.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-03-24T02:37:27.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-03-24T02:37:27.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:27.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:27.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:27.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:27.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:27.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:27.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:27.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:27.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:27.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:27.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-03-24T02:37:27.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:27.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:27.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:27.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:27.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:27.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:27.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:27.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:27.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:27.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:27.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:27.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:27.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:27.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:27.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:27.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:27.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:27.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:27.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:27.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:27.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:27.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:27.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:27.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:27.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:27.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:27.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:27.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:27.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:27.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:27.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:27.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:27.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:27.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-03-24T02:37:27.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 1 closest matches: 2026-03-24T02:37:27.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_22 2026-03-24T02:37:27.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:27.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:27.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:27.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:27.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:27.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:27.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:27.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:27.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-03-24T02:37:27.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 5 closest matches: 2026-03-24T02:37:27.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_21 2026-03-24T02:37:27.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_24 2026-03-24T02:37:27.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_25 2026-03-24T02:37:27.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_27 2026-03-24T02:37:27.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_28 2026-03-24T02:37:27.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:27.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:27.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-03-24T02:37:27.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:37:27.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:27.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:27.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:27.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:37:27.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:27.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:27.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:27.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:37:27.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:27.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:27.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:27.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:37:27.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:37:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:27.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:27.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:27.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:27.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-03-24T02:37:27.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 3 closest matches: 2026-03-24T02:37:27.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_23 2026-03-24T02:37:27.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_26 2026-03-24T02:37:27.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_29 2026-03-24T02:37:27.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:27.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:27.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_2 2026-03-24T02:37:27.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:37:27.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:27.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:27.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:37:27.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:37:27.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:27.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:27.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:27.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:37:27.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:27.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:27.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:27.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:37:27.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:37:27.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:37:27.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:37:27.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:27.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:27.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:37:27.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:37:27.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:27.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:27.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:27.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:37:27.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:27.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:27.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:27.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:37:27.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:37:27.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:37:27.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:27.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:27.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:27.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_2 2026-03-24T02:37:27.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:27.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-03-24T02:37:27.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:27.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:27.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:37:27.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-03-24T02:37:27.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:27.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:27.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:37:27.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:27.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-03-24T02:37:27.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_2.status 2026-03-24T02:37:27.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:27.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:27.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-03-24T02:37:27.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:27.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:27.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:27.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:27.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6087], [object_number=6, tag_tid=1, entry_tid=6086]]], state=connected] 2026-03-24T02:37:27.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6087], [object_number=6, tag_tid=1, entry_tid=6086], [object_number=5, tag_tid=1, entry_tid=6085], [object_number=4, tag_tid=1, entry_tid=6084]]], state=connected] 2026-03-24T02:37:27.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:27.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:27.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-03-24T02:37:27.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:27.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:27.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:37:27.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-03-24T02:37:27.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:27.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:27.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:37:27.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:27.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-03-24T02:37:27.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_2.status 2026-03-24T02:37:27.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:27.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:27.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-03-24T02:37:27.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:27.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:27.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:27.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:27.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6087], [object_number=6, tag_tid=1, entry_tid=6086]]], state=connected] 2026-03-24T02:37:27.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6087], [object_number=6, tag_tid=1, entry_tid=6086], [object_number=5, tag_tid=1, entry_tid=6085], [object_number=4, tag_tid=1, entry_tid=6084]]], state=connected] 2026-03-24T02:37:27.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:27.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=6087' 2026-03-24T02:37:27.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=6087' 2026-03-24T02:37:27.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6087' '!=' 'object_number=7, tag_tid=1, entry_tid=6087' 2026-03-24T02:37:27.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:27.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:37:28.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_2 2026-03-24T02:37:28.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:28.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:28.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:37:28.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:28.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_2 ']' 2026-03-24T02:37:28.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-03-24T02:37:28.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-03-24T02:37:28.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:28.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:28.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:28.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:28.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:28.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:28.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:28.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:28.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:28.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:28.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-03-24T02:37:28.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:28.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:28.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:28.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:28.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:28.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:28.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:28.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:28.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:28.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:28.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-03-24T02:37:28.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 1 closest matches: 2026-03-24T02:37:28.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_22 2026-03-24T02:37:28.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:28.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:28.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-03-24T02:37:28.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:37:28.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:28.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:28.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:28.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:37:28.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:28.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:28.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:28.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:28.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:28.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:28.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-03-24T02:37:28.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 5 closest matches: 2026-03-24T02:37:28.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_21 2026-03-24T02:37:28.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_24 2026-03-24T02:37:28.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_25 2026-03-24T02:37:28.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_27 2026-03-24T02:37:28.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_28 2026-03-24T02:37:28.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:28.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:28.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-03-24T02:37:28.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:37:28.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:28.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:28.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:28.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:28.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:28.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:28.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:28.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:28.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:28.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:28.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-03-24T02:37:28.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 3 closest matches: 2026-03-24T02:37:28.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_23 2026-03-24T02:37:28.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_26 2026-03-24T02:37:28.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_29 2026-03-24T02:37:28.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:28.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:28.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_2 2026-03-24T02:37:28.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:37:28.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:28.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:28.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:37:28.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:37:28.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:28.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:28.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:28.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:37:28.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:28.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:28.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:37:28.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:37:28.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:28.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:28.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:28.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_2 2026-03-24T02:37:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:28.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-03-24T02:37:28.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:28.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:28.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:37:28.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-03-24T02:37:28.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:28.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:28.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:37:28.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:28.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-03-24T02:37:28.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_2.status 2026-03-24T02:37:28.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:28.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:28.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-03-24T02:37:28.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:28.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:28.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:28.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:28.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6087], [object_number=6, tag_tid=1, entry_tid=6086]]], state=connected] 2026-03-24T02:37:28.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6087], [object_number=6, tag_tid=1, entry_tid=6086]]], state=connected] 2026-03-24T02:37:28.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:28.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:28.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-03-24T02:37:28.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:28.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:28.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:37:28.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-03-24T02:37:28.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:28.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:28.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:37:28.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:28.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-03-24T02:37:28.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_2.status 2026-03-24T02:37:28.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:28.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:28.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:28.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-03-24T02:37:28.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:28.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:28.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:28.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6087], [object_number=6, tag_tid=1, entry_tid=6086]]], state=connected] 2026-03-24T02:37:28.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6087], [object_number=6, tag_tid=1, entry_tid=6086]]], state=connected] 2026-03-24T02:37:28.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.status 2026-03-24T02:37:28.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:28.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:28.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:28.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_2 snap 2026-03-24T02:37:28.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:28.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:28.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:37:28.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:37:28.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:37:28.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:37:28.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:37:29.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_2@snap 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_2': 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c948f11ac0 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c948f11ac0 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:25:56 2026 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:25:56 2026 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:36:52 2026 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c948f11ac0 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 5a6bdb2d-91a2-42a6-85fd-a7ffe38d62cb 2026-03-24T02:37:29.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:37:29.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:29.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_2 snap 2026-03-24T02:37:29.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:29.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:37:29.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:37:29.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:37:29.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.export 2026-03-24T02:37:29.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_2.export 2026-03-24T02:37:29.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_2.export 2026-03-24T02:37:29.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_2@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.export 2026-03-24T02:37:29.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:37:29.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_2@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_2.export 2026-03-24T02:37:30.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:37:30.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_2.export 2026-03-24T02:37:30.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_2.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_2.export 2026-03-24T02:37:30.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:30.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:37:30.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_3 2026-03-24T02:37:30.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_3 snap 2026-03-24T02:37:30.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:30.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:30.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:37:30.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:37:30.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_3@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:37:30.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_3 2026-03-24T02:37:30.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:30.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:30.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:37:30.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_3 started 2026-03-24T02:37:30.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:30.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:30.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:37:30.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:37:30.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:37:30.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:30.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:37:31.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_3 started 2026-03-24T02:37:31.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:31.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:31.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:37:31.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:37:31.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:37:31.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:37:31.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_3 2026-03-24T02:37:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:37:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:37:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:37:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:37:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:37:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:37:31.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:31.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_3 2026-03-24T02:37:31.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:37:31.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:31.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:31.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:31.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:31.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:37:31.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:31.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:31.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:31.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:31.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:37:31.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:31.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:31.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:31.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:31.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:37:31.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:37:31.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:37:31.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:31.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:37:31.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:37:31.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:37:31.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:31.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:31.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:31.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:37:31.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:37:31.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:37:31.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:37:31.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:31.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:31.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:31.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:31.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_3 2026-03-24T02:37:32.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 4 closest matches: 2026-03-24T02:37:32.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status mirror/image_34 2026-03-24T02:37:32.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status mirror/image_35 2026-03-24T02:37:32.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status mirror/image_36 2026-03-24T02:37:32.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status mirror/image_38 2026-03-24T02:37:32.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_3 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:32.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:32.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_3 2026-03-24T02:37:32.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 2 closest matches: 2026-03-24T02:37:32.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status mirror/image_33 2026-03-24T02:37:32.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status mirror/image_37 2026-03-24T02:37:32.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_3 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:32.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_3 2026-03-24T02:37:32.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 2 closest matches: 2026-03-24T02:37:32.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status mirror/image_31 2026-03-24T02:37:32.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status mirror/image_39 2026-03-24T02:37:32.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:32.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:32.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_3 2026-03-24T02:37:32.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:37:32.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:32.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:32.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:37:32.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:37:32.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:37:32.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:32.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:32.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:37:32.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:32.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:37:32.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:32.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:37:32.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:37:32.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:37:32.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:37:32.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:37:32.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:37:32.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:37:32.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:37:32.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:37:32.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:37:32.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:32.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:37:32.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:32.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:37:32.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:37:32.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:37:32.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:37:32.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:37:32.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:32.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:32.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:32.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_3 2026-03-24T02:37:32.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:37:32.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:37:32.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:37:32.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_3 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_3 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:37:32.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:37:32.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:37:32.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:32.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:37:32.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_3 2026-03-24T02:37:32.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:32.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:32.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:37:32.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:32.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_3 ']' 2026-03-24T02:37:32.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-03-24T02:37:32.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-03-24T02:37:32.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:32.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:32.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:32.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:32.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:32.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:32.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:32.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:32.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:32.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:32.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-03-24T02:37:32.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:32.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:32.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:32.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:32.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:32.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:32.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:32.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:32.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:32.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:32.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:32.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:32.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:32.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:32.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:32.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:32.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:32.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:32.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:32.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:32.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:32.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:32.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:32.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:32.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:32.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:32.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:32.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:32.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:32.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:32.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:32.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:32.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:32.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-03-24T02:37:32.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 4 closest matches: 2026-03-24T02:37:32.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_34 2026-03-24T02:37:32.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_35 2026-03-24T02:37:32.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_36 2026-03-24T02:37:32.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_38 2026-03-24T02:37:32.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:32.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:32.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:32.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:32.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:32.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:32.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:32.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:32.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2026-03-24T02:37:32.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 2 closest matches: 2026-03-24T02:37:32.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_33 2026-03-24T02:37:32.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_37 2026-03-24T02:37:32.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_3 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:32.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:32.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:32.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:32.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:32.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:32.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:32.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:32.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:32.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:32.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:32.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:32.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:32.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:32.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_3 2026-03-24T02:37:32.693 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 2 closest matches: 2026-03-24T02:37:32.693 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_31 2026-03-24T02:37:32.693 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_39 2026-03-24T02:37:32.693 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:32.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_3 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:37:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:37:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_3 2026-03-24T02:37:32.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:32.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-03-24T02:37:32.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:32.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:32.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:37:32.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-03-24T02:37:32.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:32.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:32.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:37:32.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:32.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-03-24T02:37:32.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_3.status 2026-03-24T02:37:32.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:32.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:32.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-03-24T02:37:32.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:32.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:37:32.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:37:32.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:32.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:37:32.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:37:32.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-03-24T02:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-03-24T02:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-03-24T02:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_3.status 2026-03-24T02:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-03-24T02:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:32.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:37:32.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:37:32.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:32.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:37:32.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:32.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:37:33.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_3 2026-03-24T02:37:33.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:33.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:33.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:37:33.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:33.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_3 ']' 2026-03-24T02:37:33.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-03-24T02:37:33.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-03-24T02:37:33.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:33.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:33.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:33.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:33.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:33.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:33.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:33.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:33.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:33.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:33.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-03-24T02:37:33.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:33.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:33.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:33.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:33.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:33.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:33.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:33.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:33.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:33.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:33.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:33.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:33.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:33.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:33.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:33.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:33.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:33.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:33.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:33.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:33.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:33.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:33.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:33.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:33.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:33.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:33.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:33.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:33.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:33.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:33.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:33.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:33.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:33.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-03-24T02:37:33.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 4 closest matches: 2026-03-24T02:37:33.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_34 2026-03-24T02:37:33.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_35 2026-03-24T02:37:33.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_36 2026-03-24T02:37:33.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_38 2026-03-24T02:37:33.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:33.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:33.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:33.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:33.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:33.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2026-03-24T02:37:33.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 2 closest matches: 2026-03-24T02:37:33.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_33 2026-03-24T02:37:33.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_37 2026-03-24T02:37:33.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_3 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:33.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_3 2026-03-24T02:37:33.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 2 closest matches: 2026-03-24T02:37:33.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_31 2026-03-24T02:37:33.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_39 2026-03-24T02:37:33.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_3 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:33.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_3 2026-03-24T02:37:33.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:33.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-03-24T02:37:33.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:33.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:33.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:37:33.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-03-24T02:37:33.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:33.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:33.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:37:33.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:33.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-03-24T02:37:33.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_3.status 2026-03-24T02:37:33.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-03-24T02:37:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:33.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:37:33.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:37:33.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:33.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:37:33.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:37:33.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:33.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:37:33.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-03-24T02:37:33.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:33.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:33.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:37:33.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-03-24T02:37:33.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:33.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:33.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:37:33.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:33.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-03-24T02:37:33.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_3.status 2026-03-24T02:37:33.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:33.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:33.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-03-24T02:37:33.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:33.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:37:33.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:37:33.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:33.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:37:33.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:37:33.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:33.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:37:33.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:37:33.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:37:33.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:33.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:37:33.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_3 2026-03-24T02:37:33.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:33.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:33.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:37:33.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:33.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_3 ']' 2026-03-24T02:37:33.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-03-24T02:37:33.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-03-24T02:37:33.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:33.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:33.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:33.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:33.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:33.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:33.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:33.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:33.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:33.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:33.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-03-24T02:37:33.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:33.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:33.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:33.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:33.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:33.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:33.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:33.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:33.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:33.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-03-24T02:37:34.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 4 closest matches: 2026-03-24T02:37:34.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_34 2026-03-24T02:37:34.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_35 2026-03-24T02:37:34.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_36 2026-03-24T02:37:34.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_38 2026-03-24T02:37:34.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:34.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:34.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2026-03-24T02:37:34.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:37:34.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:34.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:34.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:34.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:37:34.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:34.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:34.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:34.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:37:34.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:34.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:34.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:34.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:37:34.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:34.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:34.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:37:34.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:34.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:34.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:34.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:34.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:34.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:34.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:34.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:34.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:34.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:34.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:34.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:34.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:34.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:34.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:34.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:34.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:34.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2026-03-24T02:37:34.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 2 closest matches: 2026-03-24T02:37:34.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_33 2026-03-24T02:37:34.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_37 2026-03-24T02:37:34.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:34.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:34.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_3 2026-03-24T02:37:34.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:37:34.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:34.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:34.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:34.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:37:34.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:34.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:34.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:34.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:37:34.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:34.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:34.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:34.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:34.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:34.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:34.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_3 2026-03-24T02:37:34.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 2 closest matches: 2026-03-24T02:37:34.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_31 2026-03-24T02:37:34.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_39 2026-03-24T02:37:34.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_3 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:34.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:34.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:37:34.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:34.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:34.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:34.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:37:34.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:37:34.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:37:34.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:34.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:34.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:34.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_3 2026-03-24T02:37:34.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:34.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-03-24T02:37:34.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:34.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:34.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:37:34.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-03-24T02:37:34.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:34.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:34.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:37:34.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:34.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-03-24T02:37:34.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_3.status 2026-03-24T02:37:34.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:34.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:34.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-03-24T02:37:34.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:34.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:37:34.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:37:34.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:34.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:37:34.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:37:34.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:34.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:37:34.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-03-24T02:37:34.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:34.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:34.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:37:34.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-03-24T02:37:34.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:34.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:34.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:37:34.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:34.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-03-24T02:37:34.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_3.status 2026-03-24T02:37:34.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:34.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:34.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-03-24T02:37:34.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:34.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:37:34.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:37:34.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:34.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:37:34.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:37:34.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:34.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:37:34.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:37:34.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:37:34.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:34.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:37:35.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_3 2026-03-24T02:37:35.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:35.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:35.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:37:35.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:35.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_3 ']' 2026-03-24T02:37:35.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-03-24T02:37:35.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-03-24T02:37:35.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:35.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:35.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:35.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:35.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:35.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:35.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:35.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:35.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-03-24T02:37:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:35.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:35.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:35.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:35.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-03-24T02:37:35.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 4 closest matches: 2026-03-24T02:37:35.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_34 2026-03-24T02:37:35.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_35 2026-03-24T02:37:35.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_36 2026-03-24T02:37:35.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_38 2026-03-24T02:37:35.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:35.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:35.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2026-03-24T02:37:35.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:37:35.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:35.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:35.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:35.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:37:35.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:35.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:35.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:35.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:37:35.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:35.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:35.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:35.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:37:35.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:35.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:35.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:37:35.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:35.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:35.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:35.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:35.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:35.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:35.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:35.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:35.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:35.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:35.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:35.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:35.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:35.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:35.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:35.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:35.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:35.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2026-03-24T02:37:35.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 2 closest matches: 2026-03-24T02:37:35.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_33 2026-03-24T02:37:35.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_37 2026-03-24T02:37:35.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:35.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:35.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_3 2026-03-24T02:37:35.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:37:35.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:35.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:35.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:35.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:37:35.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:35.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:35.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:35.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:37:35.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:35.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:35.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:35.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:37:35.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:35.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:35.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:37:35.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:35.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:35.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:35.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:35.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:35.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:35.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:35.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:35.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:35.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:35.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:35.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:35.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:35.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:35.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:35.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:35.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:35.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_3 2026-03-24T02:37:35.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:no valid command found; 2 closest matches: 2026-03-24T02:37:35.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_31 2026-03-24T02:37:35.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush mirror/image_39 2026-03-24T02:37:35.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:35.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:35.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_3 2026-03-24T02:37:35.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:37:35.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:35.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:37:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:37:35.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:35.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:35.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:37:35.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_3 2026-03-24T02:37:35.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:35.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-03-24T02:37:35.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:35.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:35.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:37:35.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-03-24T02:37:35.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:35.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:35.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:37:35.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:35.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-03-24T02:37:35.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_3.status 2026-03-24T02:37:35.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:35.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:35.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-03-24T02:37:35.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:35.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:37:35.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:37:35.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:35.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:37:35.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:37:35.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:35.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:37:35.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-03-24T02:37:35.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:35.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:35.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:37:35.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-03-24T02:37:35.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:35.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:35.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:37:35.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:35.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-03-24T02:37:35.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_3.status 2026-03-24T02:37:35.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:35.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:35.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-03-24T02:37:35.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:35.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:37:35.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:37:35.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:35.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:37:35.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:37:35.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.status 2026-03-24T02:37:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:37:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:37:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_3 snap 2026-03-24T02:37:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:37:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:37:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:37:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:37:35.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:37:36.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_3@snap 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_3': 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 2 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 116d7c25fbc3 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.116d7c25fbc3 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:26:02 2026 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:26:02 2026 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:27:03 2026 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 116d7c25fbc3 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 5db72427-62e1-4fc5-b013-bc5a96b062f4 2026-03-24T02:37:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:37:36.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:36.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_3 snap 2026-03-24T02:37:36.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:36.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:37:36.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:37:36.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:37:36.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.export 2026-03-24T02:37:36.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_3.export 2026-03-24T02:37:36.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_3.export 2026-03-24T02:37:36.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_3@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.export 2026-03-24T02:37:37.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:37:37.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_3@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_3.export 2026-03-24T02:37:37.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:37:37.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_3.export 2026-03-24T02:37:37.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_3.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_3.export 2026-03-24T02:37:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:37:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_4 2026-03-24T02:37:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_4 snap 2026-03-24T02:37:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:37:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:37:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_4@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:37:38.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_4 2026-03-24T02:37:38.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:38.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:38.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:37:38.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_4 started 2026-03-24T02:37:38.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:38.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:38.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:37:38.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:37:38.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:37:38.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:38.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:37:39.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_4 started 2026-03-24T02:37:39.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:39.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:39.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:37:39.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:37:39.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:37:39.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:37:39.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_4 2026-03-24T02:37:39.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:37:39.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:37:39.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:39.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:39.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:37:39.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:37:39.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:39.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:37:39.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:39.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:37:39.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_4 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:37:39.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:37:39.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:39.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:39.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:39.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:37:39.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:37:39.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:37:39.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:37:39.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:39.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:39.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:39.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:39.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_4 2026-03-24T02:37:39.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:37:39.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:37:39.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:37:39.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:37:39.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:37:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:37:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:37:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_4 2026-03-24T02:37:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:37:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:37:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:37:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:37:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:37:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_4 2026-03-24T02:37:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:37:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:37:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:37:39.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:37:39.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:37:39.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:37:39.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:37:39.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:39.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:37:39.915 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7972604539774923) > weight (0.25) 2026-03-24T02:37:39.916 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6465592136771838) > weight (0.25) 2026-03-24T02:37:39.916 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-03-24T02:37:39.916 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-03-24T02:37:39.916 INFO:tasks.rbd_mirror_thrash:waiting for 4.022899047574743 secs before reviving daemons 2026-03-24T02:37:39.916 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:37:39.912+0000 7f5e3532e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 51362) UID: 1000 2026-03-24T02:37:39.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_4 2026-03-24T02:37:39.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:39.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:39.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:37:39.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:39.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_4 ']' 2026-03-24T02:37:39.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_4' 2026-03-24T02:37:39.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_4 2026-03-24T02:37:39.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:39.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:39.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:39.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:39.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:39.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:39.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:39.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:39.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_4 2026-03-24T02:37:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:39.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:39.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:39.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:39.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_4 2026-03-24T02:37:40.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:40.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_4 2026-03-24T02:37:40.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:40.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:40.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_4 2026-03-24T02:37:40.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_4 '' 2026-03-24T02:37:40.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:40.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:40.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_4 2026-03-24T02:37:40.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:40.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-03-24T02:37:40.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_4.status 2026-03-24T02:37:40.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:40.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_4.status 2026-03-24T02:37:40.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-03-24T02:37:40.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_4.status 2026-03-24T02:37:40.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:40.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:40.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:40.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5695], [object_number=6, tag_tid=1, entry_tid=5694]]], state=connected] 2026-03-24T02:37:40.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5695], [object_number=6, tag_tid=1, entry_tid=5694]]], state=connected] 2026-03-24T02:37:40.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_4.status 2026-03-24T02:37:40.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:40.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_4 2026-03-24T02:37:40.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:40.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:40.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_4 2026-03-24T02:37:40.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_4 '..*' 2026-03-24T02:37:40.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:40.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:40.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_4 2026-03-24T02:37:40.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:40.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-03-24T02:37:40.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_4.status 2026-03-24T02:37:40.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:40.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_4.status 2026-03-24T02:37:40.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-03-24T02:37:40.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_4.status 2026-03-24T02:37:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5695], [object_number=6, tag_tid=1, entry_tid=5694]]], state=connected] 2026-03-24T02:37:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5695], [object_number=6, tag_tid=1, entry_tid=5694]]], state=connected] 2026-03-24T02:37:40.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_4.status 2026-03-24T02:37:40.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:40.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:40.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:40.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_4 snap 2026-03-24T02:37:40.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:40.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:40.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:37:40.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:37:40.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:37:40.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:37:40.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:37:41.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_4@snap 2026-03-24T02:37:41.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_4': 2026-03-24T02:37:41.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:37:41.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:37:41.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:37:41.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 119ee0884471 2026-03-24T02:37:41.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.119ee0884471 2026-03-24T02:37:41.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:37:41.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:37:41.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:37:41.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:37:41.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:26:08 2026 2026-03-24T02:37:41.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:26:08 2026 2026-03-24T02:37:41.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:36:50 2026 2026-03-24T02:37:41.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:37:41.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 119ee0884471 2026-03-24T02:37:41.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:37:41.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:37:41.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 47d8807b-cf84-4f59-9a60-b46ffdf224d4 2026-03-24T02:37:41.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:37:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_4 snap 2026-03-24T02:37:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:37:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:37:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:37:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_4.export 2026-03-24T02:37:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_4.export 2026-03-24T02:37:41.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_4.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_4.export 2026-03-24T02:37:41.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_4@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_4.export 2026-03-24T02:37:41.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:37:41.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_4@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_4.export 2026-03-24T02:37:41.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:37:41.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_4.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_4.export 2026-03-24T02:37:41.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_4.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_4.export 2026-03-24T02:37:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:41.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:37:41.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_5 2026-03-24T02:37:41.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_5 snap 2026-03-24T02:37:41.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:41.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:41.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:37:41.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:37:41.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_5@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:37:42.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_5 2026-03-24T02:37:42.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:42.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:42.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:37:42.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_5 started 2026-03-24T02:37:42.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:42.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:42.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:37:42.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:37:42.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:37:42.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:42.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:37:43.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_5 started 2026-03-24T02:37:43.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:43.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:43.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:37:43.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:37:43.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:37:43.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:37:43.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_5 2026-03-24T02:37:43.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:37:43.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:37:43.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:43.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:43.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:37:43.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:37:43.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:43.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:37:43.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:43.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:37:43.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:43.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_5 2026-03-24T02:37:43.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:37:43.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:43.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:43.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:43.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:43.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:37:43.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:43.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:43.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:43.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:43.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:37:43.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:43.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:43.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:43.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:43.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:37:43.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:37:43.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:37:43.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:43.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:37:43.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:37:43.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:37:43.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:43.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:43.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:43.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:37:43.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:37:43.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:37:43.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:37:43.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:43.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:43.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:43.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:43.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_5 2026-03-24T02:37:43.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-03-24T02:37:43.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:37:43.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:43.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:43.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_5 2026-03-24T02:37:43.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:37:43.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:43.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:43.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:43.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:43.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:37:43.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:43.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:43.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:43.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:43.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:37:43.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:43.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:43.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:43.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:43.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:37:43.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:37:43.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:37:43.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:43.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:37:43.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:37:43.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:37:43.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:43.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:43.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:43.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:37:43.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:37:43.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:37:43.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:37:43.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:43.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:43.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:43.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:43.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_5 2026-03-24T02:37:43.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-03-24T02:37:43.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:37:43.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:43.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:43.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_5 2026-03-24T02:37:43.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:37:43.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:43.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:43.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:43.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:43.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:37:43.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:43.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:43.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:43.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:43.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:37:43.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:43.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:43.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:43.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:43.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:37:43.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:37:43.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:37:43.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:43.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:37:43.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:37:43.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:37:43.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:43.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:43.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:43.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:37:43.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:37:43.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:37:43.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:37:43.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:43.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:43.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:43.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:43.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_5 2026-03-24T02:37:43.939 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-03-24T02:37:43.939 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:37:43.939 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-03-24T02:37:43.939 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-03-24T02:37:43.939 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-03-24T02:37:43.941 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-03-24T02:37:43.941 INFO:tasks.rbd_mirror_thrash:waiting for 94.55249355335394 secs before thrashing 2026-03-24T02:37:44.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:37:44.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:37:44.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:37:44.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:37:44.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_5 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_5 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:44.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_5 2026-03-24T02:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_5 ']' 2026-03-24T02:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-03-24T02:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-03-24T02:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:44.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:44.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:44.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:44.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:44.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:44.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:44.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:44.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:44.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:44.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:44.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:44.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:44.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-03-24T02:37:44.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-03-24T02:37:44.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:44.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:44.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:44.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-03-24T02:37:44.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:37:44.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:44.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:44.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:44.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:37:44.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:44.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:44.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:44.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:37:44.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:44.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:44.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:44.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:37:44.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:44.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:44.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:37:44.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:44.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:44.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:44.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:44.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:44.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:44.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:44.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:44.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:44.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:44.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:44.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:44.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:44.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:44.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:44.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:44.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:44.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-03-24T02:37:44.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-03-24T02:37:44.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:44.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:44.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:44.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2026-03-24T02:37:44.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:37:44.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:44.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:44.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:44.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:37:44.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:44.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:44.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:44.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:37:44.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:44.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:44.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:44.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:37:44.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:44.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:44.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:37:44.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:44.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:44.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:44.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:44.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:44.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:44.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:44.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:44.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:44.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:44.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:44.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:44.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:44.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:44.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2026-03-24T02:37:44.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:44.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-03-24T02:37:44.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:44.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:44.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:44.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-03-24T02:37:44.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:44.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:44.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:44.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:44.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-03-24T02:37:44.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_5.status 2026-03-24T02:37:44.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:44.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-03-24T02:37:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:44.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:44.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:44.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:44.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:44.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:44.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:44.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:44.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-03-24T02:37:44.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:44.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:44.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:44.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-03-24T02:37:44.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:44.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:44.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:44.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:44.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-03-24T02:37:44.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_5.status 2026-03-24T02:37:44.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:44.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:44.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-03-24T02:37:44.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:44.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:44.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:44.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:44.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:44.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:44.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:44.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:44.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:44.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' '' 2026-03-24T02:37:44.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:37:44.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' = '' 2026-03-24T02:37:44.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:44.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:44.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:37:44.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:44.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:37:44.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:44.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:37:44.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:44.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:37:44.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:44.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:37:44.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:44.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:37:44.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:44.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:37:44.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:44.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=1 2026-03-24T02:37:44.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 1 2026-03-24T02:37:44.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:37:44.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:44.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_5 2026-03-24T02:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_5 ']' 2026-03-24T02:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-03-24T02:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-03-24T02:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:44.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:44.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-03-24T02:37:44.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:44.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:44.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:44.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:44.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:44.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:44.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:44.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:44.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:44.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:44.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:44.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:44.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:44.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:44.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:44.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:44.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:44.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:44.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:44.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:44.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:44.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:44.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:44.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:44.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:44.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:44.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:44.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:44.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:44.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:44.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:44.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:44.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:44.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-03-24T02:37:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-03-24T02:37:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:44.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:44.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-03-24T02:37:44.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:37:44.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:44.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:44.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:44.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:37:44.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:44.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:44.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:44.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:37:44.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:44.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:44.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:44.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:37:44.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:44.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:44.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:37:44.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:44.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:44.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:44.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:44.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:44.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:44.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:44.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:44.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:44.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:44.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:44.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:44.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:44.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:44.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:44.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:44.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:44.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-03-24T02:37:45.023 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:37:45.017+0000 7f5496ce4640 -1 reset not still connected to 0x5652c92ea5b0 2026-03-24T02:37:45.023 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:37:45.017+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c5de10 2026-03-24T02:37:45.023 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:37:45.017+0000 7f5496ce4640 -1 reset not still connected to 0x5652d714f930 2026-03-24T02:37:45.031 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:37:45.025+0000 7f5496ce4640 -1 reset not still connected to 0x5652ef236ea0 2026-03-24T02:37:45.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-03-24T02:37:45.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:45.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:45.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:45.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2026-03-24T02:37:45.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:45.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:45.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:45.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:45.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:45.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:45.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:45.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:45.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:45.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:45.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2026-03-24T02:37:45.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:45.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-03-24T02:37:45.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:45.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:45.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:45.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-03-24T02:37:45.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:45.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:45.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:45.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:45.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-03-24T02:37:45.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_5.status 2026-03-24T02:37:45.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:45.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:45.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-03-24T02:37:45.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:45.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:45.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:45.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:45.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:45.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:45.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:45.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:45.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-03-24T02:37:45.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:45.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-03-24T02:37:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-03-24T02:37:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_5.status 2026-03-24T02:37:45.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:45.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-03-24T02:37:45.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:45.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:45.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:45.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:45.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:45.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:45.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:45.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:45.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:45.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:45.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:45.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:37:45.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_5 2026-03-24T02:37:45.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:45.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:45.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:37:45.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:45.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_5 ']' 2026-03-24T02:37:45.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-03-24T02:37:45.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-03-24T02:37:45.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:45.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:45.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:45.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:45.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:45.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:45.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:45.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:45.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-03-24T02:37:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:45.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:45.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:45.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:45.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:45.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:45.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:45.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:45.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:45.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:45.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:45.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:45.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:45.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:45.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:45.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:45.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:45.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:45.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:45.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:45.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-03-24T02:37:45.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-03-24T02:37:45.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:45.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:45.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:45.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-03-24T02:37:45.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:37:45.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:45.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:45.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:45.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:45.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:45.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:45.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:45.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-03-24T02:37:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-03-24T02:37:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:45.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:45.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2026-03-24T02:37:45.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:37:45.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:45.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:45.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:45.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:37:45.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:45.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:45.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:45.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:37:45.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:45.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:45.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:45.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:37:45.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:45.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:45.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:37:45.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:45.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:45.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:45.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:45.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:45.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:45.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:45.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:45.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:45.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:45.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:45.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:45.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:45.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:45.832 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:45.832 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:45.832 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:45.832 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2026-03-24T02:37:45.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:45.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-03-24T02:37:45.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:45.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:45.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:45.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-03-24T02:37:45.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:45.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:45.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:45.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:45.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-03-24T02:37:45.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_5.status 2026-03-24T02:37:45.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:45.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:45.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-03-24T02:37:45.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:45.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:45.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:45.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:45.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:45.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:45.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:45.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:45.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-03-24T02:37:45.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:45.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:45.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:45.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-03-24T02:37:45.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:45.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:45.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:45.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:45.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-03-24T02:37:45.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_5.status 2026-03-24T02:37:45.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:45.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:45.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-03-24T02:37:45.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:46.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:46.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:46.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:46.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:46.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:46.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:46.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:46.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:46.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:46.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:46.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:37:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_5 2026-03-24T02:37:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:37:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_5 ']' 2026-03-24T02:37:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-03-24T02:37:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-03-24T02:37:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:46.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:46.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:46.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:46.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:46.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:46.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:46.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:46.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:46.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:46.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:46.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:46.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:46.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:46.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:46.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:46.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:46.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:46.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:46.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-03-24T02:37:46.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-03-24T02:37:46.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:46.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:46.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:46.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:46.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:46.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:46.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:46.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:46.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:46.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:46.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:46.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:46.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:46.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-03-24T02:37:46.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-03-24T02:37:46.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:46.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:46.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:46.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:46.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:46.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:46.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:46.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:46.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:46.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:46.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:46.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2026-03-24T02:37:47.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:47.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-03-24T02:37:47.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:47.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:47.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:47.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-03-24T02:37:47.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:47.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:47.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:47.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:47.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-03-24T02:37:47.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_5.status 2026-03-24T02:37:47.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:47.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:47.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-03-24T02:37:47.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:47.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:47.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:47.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:47.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:47.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:47.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:47.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:47.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-03-24T02:37:47.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:47.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:47.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:47.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-03-24T02:37:47.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:47.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:47.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:47.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:47.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-03-24T02:37:47.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_5.status 2026-03-24T02:37:47.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-03-24T02:37:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:47.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:47.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:47.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:47.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:47.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:47.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:47.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:47.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:47.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:47.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:47.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:37:48.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_5 2026-03-24T02:37:48.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:48.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:48.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:37:48.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:48.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_5 ']' 2026-03-24T02:37:48.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-03-24T02:37:48.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-03-24T02:37:48.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:48.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:48.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:48.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:48.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:48.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:48.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:48.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:48.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:48.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:48.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-03-24T02:37:48.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:48.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:48.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:48.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:48.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:48.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:48.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:48.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:48.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:48.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:48.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:48.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:48.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:48.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:48.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:48.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:48.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:48.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:48.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:48.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:48.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:48.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:48.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:48.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:48.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:48.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:48.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:48.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:48.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:48.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:48.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:48.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:48.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:48.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-03-24T02:37:48.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-03-24T02:37:48.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:48.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:48.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:48.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-03-24T02:37:48.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:37:48.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:48.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:48.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:48.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:37:48.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:48.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:48.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:48.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:37:48.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:48.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:48.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:48.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:37:48.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:48.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:48.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:37:48.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:48.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:48.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:48.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:48.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:48.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:48.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:48.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:48.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:48.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:48.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:48.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:48.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:48.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:48.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:48.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:48.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:48.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-03-24T02:37:48.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-03-24T02:37:48.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:48.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:48.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:48.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2026-03-24T02:37:48.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:37:48.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:48.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:48.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:48.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:37:48.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:48.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:48.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:48.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:37:48.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:48.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:48.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:48.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:37:48.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:48.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:48.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:37:48.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:48.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:48.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:48.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:48.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:48.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:48.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:48.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:48.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:48.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:48.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:48.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:48.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:48.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2026-03-24T02:37:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-03-24T02:37:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-03-24T02:37:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:49.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:49.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-03-24T02:37:49.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_5.status 2026-03-24T02:37:49.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-03-24T02:37:49.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:49.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:49.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:49.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-03-24T02:37:49.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:49.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:49.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:49.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-03-24T02:37:49.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:49.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:49.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:37:49.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:49.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-03-24T02:37:49.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_5.status 2026-03-24T02:37:49.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:49.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:49.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-03-24T02:37:49.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:49.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:49.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:49.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:49.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:49.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5187], [object_number=6, tag_tid=1, entry_tid=5186]]], state=connected] 2026-03-24T02:37:49.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.status 2026-03-24T02:37:49.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:49.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:37:49.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:49.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_5 snap 2026-03-24T02:37:49.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:49.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:49.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:37:49.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:37:49.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:37:49.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:37:49.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:37:50.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_5@snap 2026-03-24T02:37:50.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_5': 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c9f562a10c 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c9f562a10c 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:26:15 2026 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:26:15 2026 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:27:26 2026 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c9f562a10c 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 7b9fe739-b5d6-465d-9c6a-447092668ba5 2026-03-24T02:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:37:50.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:50.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_5 snap 2026-03-24T02:37:50.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:50.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:37:50.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:37:50.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:37:50.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.export 2026-03-24T02:37:50.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_5.export 2026-03-24T02:37:50.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_5.export 2026-03-24T02:37:50.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_5@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.export 2026-03-24T02:37:50.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:37:50.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_5@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_5.export 2026-03-24T02:37:50.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:37:50.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_5.export 2026-03-24T02:37:50.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_5.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_5.export 2026-03-24T02:37:50.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:50.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:37:50.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_6 2026-03-24T02:37:50.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_6 snap 2026-03-24T02:37:50.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:50.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:50.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:37:50.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:37:50.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_6@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:37:51.538 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:37:51.529+0000 7f5496ce4640 -1 reset not still connected to 0x5652d714f930 2026-03-24T02:37:51.538 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:37:51.529+0000 7f5496ce4640 -1 reset not still connected to 0x5652ef236ea0 2026-03-24T02:37:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_6 2026-03-24T02:37:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:37:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_6 started 2026-03-24T02:37:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:37:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:37:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:37:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:37:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_6 started 2026-03-24T02:37:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:37:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:37:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:37:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:37:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_6 2026-03-24T02:37:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:37:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:37:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:37:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:37:53.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:53.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:37:53.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:53.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:37:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_6 2026-03-24T02:37:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:37:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:37:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:37:53.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:53.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:53.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:53.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:53.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_6 2026-03-24T02:37:53.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-03-24T02:37:53.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:37:53.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:53.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:53.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_6 2026-03-24T02:37:53.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:37:53.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:53.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:53.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:53.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:53.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:37:53.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:53.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:53.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:37:53.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:53.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:53.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:53.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:53.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_6 2026-03-24T02:37:53.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-03-24T02:37:53.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:37:53.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:53.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:53.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_6 2026-03-24T02:37:53.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:37:53.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:37:53.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:53.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:53.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:53.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:37:53.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:53.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:53.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:53.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:53.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:37:53.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:53.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:53.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:53.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:53.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:37:53.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:37:53.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:37:53.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:53.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:37:53.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:37:53.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:37:53.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:37:53.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:53.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:53.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:37:53.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:37:53.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:37:53.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:37:53.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:53.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:53.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:53.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:53.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_6 2026-03-24T02:37:53.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:37:53.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:37:53.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:37:53.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_6 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_6 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:37:53.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_6 2026-03-24T02:37:53.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:53.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:53.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:37:53.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:53.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_6 ']' 2026-03-24T02:37:53.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-03-24T02:37:53.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-03-24T02:37:53.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:53.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:53.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:53.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:53.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:53.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:53.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:53.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:53.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-03-24T02:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:53.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:53.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:53.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:53.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-03-24T02:37:53.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-03-24T02:37:53.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:53.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-03-24T02:37:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:37:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:37:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:37:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:37:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:37:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:53.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:53.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:53.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:53.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-03-24T02:37:54.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-03-24T02:37:54.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:54.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_6 2026-03-24T02:37:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:37:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:37:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:37:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:54.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:54.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:54.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:54.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:54.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_6 2026-03-24T02:37:54.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:54.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-03-24T02:37:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-03-24T02:37:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-03-24T02:37:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_6.status 2026-03-24T02:37:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:54.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:54.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-03-24T02:37:54.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:54.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:54.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:54.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:54.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942]]], state=connected] 2026-03-24T02:37:54.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=4, tag_tid=1, entry_tid=7944], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942], [object_number=5, tag_tid=1, entry_tid=7941]]], state=connected] 2026-03-24T02:37:54.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:54.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:54.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-03-24T02:37:54.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:54.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:54.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:54.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-03-24T02:37:54.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:54.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:54.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:54.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:54.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-03-24T02:37:54.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_6.status 2026-03-24T02:37:54.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-03-24T02:37:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:54.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:54.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:54.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:54.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942]]], state=connected] 2026-03-24T02:37:54.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=4, tag_tid=1, entry_tid=7944], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942], [object_number=5, tag_tid=1, entry_tid=7941]]], state=connected] 2026-03-24T02:37:54.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:54.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=4, tag_tid=1, entry_tid=7944' 2026-03-24T02:37:54.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=4, tag_tid=1, entry_tid=7944' 2026-03-24T02:37:54.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=4, tag_tid=1, entry_tid=7944' '!=' '' 2026-03-24T02:37:54.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:37:54.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=4, tag_tid=1, entry_tid=7944' = '' 2026-03-24T02:37:54.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=4, tag_tid=1, entry_tid=7944' 2026-03-24T02:37:54.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:54.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:37:54.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:54.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=2 2026-03-24T02:37:54.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=4, tag_tid=1, entry_tid=7944' 2026-03-24T02:37:54.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:37:54.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:54.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=1 2026-03-24T02:37:54.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:54.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:37:54.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:54.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:37:54.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=4, tag_tid=1, entry_tid=7944' 2026-03-24T02:37:54.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:37:54.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:37:54.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=7944 2026-03-24T02:37:54.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 2 = 1 -a 1 -le 7944 2026-03-24T02:37:54.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:37:54.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:54.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:37:54.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_6 2026-03-24T02:37:54.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:54.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:54.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:37:54.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:54.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_6 ']' 2026-03-24T02:37:54.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-03-24T02:37:54.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-03-24T02:37:54.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:54.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:54.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:54.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:54.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:54.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:54.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:54.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:54.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:54.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:54.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:54.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:54.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:54.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:54.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:54.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:54.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:54.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:54.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:54.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:54.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:54.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:54.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:54.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:54.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-03-24T02:37:54.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-03-24T02:37:54.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:54.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:54.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:54.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-03-24T02:37:54.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:37:54.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:54.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:54.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:54.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:54.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:54.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:54.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:54.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:54.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-03-24T02:37:54.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-03-24T02:37:54.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:54.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_6 2026-03-24T02:37:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:37:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:37:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:37:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:37:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:37:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:54.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:54.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:54.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:54.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_6 2026-03-24T02:37:54.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:54.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-03-24T02:37:54.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:54.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:54.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:54.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-03-24T02:37:54.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:54.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:54.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:54.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:54.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-03-24T02:37:54.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_6.status 2026-03-24T02:37:54.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:54.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:54.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-03-24T02:37:54.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:54.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:54.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:54.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:54.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942]]], state=connected] 2026-03-24T02:37:54.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=4, tag_tid=1, entry_tid=7944], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942], [object_number=5, tag_tid=1, entry_tid=7941]]], state=connected] 2026-03-24T02:37:54.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:54.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-03-24T02:37:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-03-24T02:37:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-03-24T02:37:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_6.status 2026-03-24T02:37:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:54.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:54.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-03-24T02:37:54.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:54.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:54.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:54.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:54.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942]]], state=connected] 2026-03-24T02:37:54.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=4, tag_tid=1, entry_tid=7944], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942], [object_number=5, tag_tid=1, entry_tid=7941]]], state=connected] 2026-03-24T02:37:54.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:54.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=4, tag_tid=1, entry_tid=7944' 2026-03-24T02:37:54.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=4, tag_tid=1, entry_tid=7944' 2026-03-24T02:37:54.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=4, tag_tid=1, entry_tid=7944' '!=' 'object_number=4, tag_tid=1, entry_tid=7944' 2026-03-24T02:37:54.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:54.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:37:55.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_6 2026-03-24T02:37:55.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:55.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:55.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:37:55.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:55.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_6 ']' 2026-03-24T02:37:55.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-03-24T02:37:55.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-03-24T02:37:55.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:55.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:55.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:55.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:55.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:55.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:55.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:55.280 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:55.280 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-03-24T02:37:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:55.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:55.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:55.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:55.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:55.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:55.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-03-24T02:37:55.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-03-24T02:37:55.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:55.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:55.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:55.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:55.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-03-24T02:37:55.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-03-24T02:37:55.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:55.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_6 2026-03-24T02:37:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:37:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:37:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:37:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:37:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:37:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:55.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:55.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:55.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:55.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_6 2026-03-24T02:37:55.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:55.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-03-24T02:37:55.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:55.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:55.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:55.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-03-24T02:37:55.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:55.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:55.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:55.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:55.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-03-24T02:37:55.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_6.status 2026-03-24T02:37:55.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:55.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:55.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-03-24T02:37:55.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:55.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:55.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:55.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:55.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942]]], state=connected] 2026-03-24T02:37:55.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=4, tag_tid=1, entry_tid=7944], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942], [object_number=5, tag_tid=1, entry_tid=7941]]], state=connected] 2026-03-24T02:37:55.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:55.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:55.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-03-24T02:37:55.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:55.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:55.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:55.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-03-24T02:37:55.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:55.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:55.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:55.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:55.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-03-24T02:37:55.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_6.status 2026-03-24T02:37:55.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:55.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:55.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-03-24T02:37:55.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:55.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:55.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:55.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:55.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942]]], state=connected] 2026-03-24T02:37:55.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=4, tag_tid=1, entry_tid=7944], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942], [object_number=5, tag_tid=1, entry_tid=7941]]], state=connected] 2026-03-24T02:37:55.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:55.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=4, tag_tid=1, entry_tid=7944' 2026-03-24T02:37:55.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=4, tag_tid=1, entry_tid=7944' 2026-03-24T02:37:55.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=4, tag_tid=1, entry_tid=7944' '!=' 'object_number=4, tag_tid=1, entry_tid=7944' 2026-03-24T02:37:55.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:55.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:37:56.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_6 2026-03-24T02:37:56.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:56.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:56.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:37:56.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:56.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_6 ']' 2026-03-24T02:37:56.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-03-24T02:37:56.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-03-24T02:37:56.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:56.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:56.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:56.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:56.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:56.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:56.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:56.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:56.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:56.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:56.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:56.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:56.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:56.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:56.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:56.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-03-24T02:37:56.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-03-24T02:37:56.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:56.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:56.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:56.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-03-24T02:37:56.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:37:56.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:56.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:56.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:56.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:37:56.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:56.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:56.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:56.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:37:56.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:56.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:56.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:56.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:37:56.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:56.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:56.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:37:56.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:56.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:56.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:56.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:56.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:56.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:56.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:56.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:56.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:56.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:56.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:56.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:56.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:56.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:56.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:56.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:56.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:56.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-03-24T02:37:56.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-03-24T02:37:56.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:56.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:56.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:56.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_6 2026-03-24T02:37:56.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:37:56.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:56.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:56.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:56.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:37:56.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:56.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:56.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:56.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:56.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:56.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:56.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:56.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:56.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:56.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:56.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_6 2026-03-24T02:37:56.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:56.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-03-24T02:37:56.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:56.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:56.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:56.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-03-24T02:37:56.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:56.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:56.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:56.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:56.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-03-24T02:37:56.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_6.status 2026-03-24T02:37:56.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:56.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:56.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-03-24T02:37:56.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:56.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:56.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:56.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:56.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942]]], state=connected] 2026-03-24T02:37:56.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=4, tag_tid=1, entry_tid=7944], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942], [object_number=5, tag_tid=1, entry_tid=7941]]], state=connected] 2026-03-24T02:37:56.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:56.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:56.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-03-24T02:37:56.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:56.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:56.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:56.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-03-24T02:37:56.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:56.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:56.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:56.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:56.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-03-24T02:37:56.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_6.status 2026-03-24T02:37:56.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:56.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:56.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:56.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-03-24T02:37:56.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:56.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:56.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:56.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942]]], state=connected] 2026-03-24T02:37:56.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=4, tag_tid=1, entry_tid=7944], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942], [object_number=5, tag_tid=1, entry_tid=7941]]], state=connected] 2026-03-24T02:37:56.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:56.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=4, tag_tid=1, entry_tid=7944' 2026-03-24T02:37:56.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=4, tag_tid=1, entry_tid=7944' 2026-03-24T02:37:56.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=4, tag_tid=1, entry_tid=7944' '!=' 'object_number=4, tag_tid=1, entry_tid=7944' 2026-03-24T02:37:56.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:56.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:37:58.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_6 2026-03-24T02:37:58.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:58.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:37:58.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:37:58.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:37:58.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_6 ']' 2026-03-24T02:37:58.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-03-24T02:37:58.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-03-24T02:37:58.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:37:58.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:58.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:37:58.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:37:58.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:37:58.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:37:58.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:37:58.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:37:58.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:37:58.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:58.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-03-24T02:37:58.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:37:58.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:58.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:58.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:37:58.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:37:58.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:58.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:58.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:58.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:37:58.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:58.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:58.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:58.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:37:58.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:58.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:37:58.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:37:58.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:58.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:58.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:37:58.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:37:58.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:58.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:58.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:58.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:37:58.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:37:58.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:58.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:58.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:37:58.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:37:58.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:37:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:37:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-03-24T02:37:58.556 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:37:58.549+0000 7fc159ee0640 -1 reset not still connected to 0x56417e130a90 2026-03-24T02:37:58.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-03-24T02:37:58.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:58.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:58.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:58.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-03-24T02:37:58.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:37:58.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:58.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:58.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:37:58.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:37:58.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:58.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:58.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:58.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:37:58.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:58.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:58.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:58.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:37:58.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:58.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:37:58.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:37:58.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:58.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:58.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:37:58.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:37:58.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:58.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:58.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:58.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:37:58.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:37:58.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:58.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:58.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:37:58.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:37:58.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:37:58.564 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:37:58.557+0000 7fc159ee0640 -1 reset not still connected to 0x5641874aab60 2026-03-24T02:37:58.572 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:37:58.565+0000 7fc159ee0640 -1 reset not still connected to 0x5641885a61a0 2026-03-24T02:37:58.574 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:37:58.569+0000 7fc159ee0640 -1 reset not still connected to 0x5641885a7110 2026-03-24T02:37:58.575 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:37:58.569+0000 7fc159ee0640 -1 reset not still connected to 0x564190e6e680 2026-03-24T02:37:58.576 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:37:58.569+0000 7fc159ee0640 -1 reset not still connected to 0x564199041860 2026-03-24T02:37:58.576 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:37:58.569+0000 7fc159ee0640 -1 reset not still connected to 0x564199041ee0 2026-03-24T02:37:58.576 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:37:58.569+0000 7fc159ee0640 -1 reset not still connected to 0x56419adcc270 2026-03-24T02:37:58.578 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:37:58.573+0000 7fc159ee0640 -1 reset not still connected to 0x5641a41df790 2026-03-24T02:37:58.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:58.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:58.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:37:58.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-03-24T02:37:58.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-03-24T02:37:58.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:37:58.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_6 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:37:58.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:37:58.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:37:58.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:37:58.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:37:58.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:37:58.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:37:58.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:37:58.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:37:58.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:37:58.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:37:58.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:37:58.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:37:58.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:58.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:58.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:37:58.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_6 2026-03-24T02:37:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:37:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-03-24T02:37:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-03-24T02:37:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:37:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-03-24T02:37:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_6.status 2026-03-24T02:37:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:58.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:58.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-03-24T02:37:58.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:58.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:58.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:58.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:58.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942]]], state=connected] 2026-03-24T02:37:58.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=4, tag_tid=1, entry_tid=7944], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942], [object_number=5, tag_tid=1, entry_tid=7941]]], state=connected] 2026-03-24T02:37:58.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:58.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:37:58.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-03-24T02:37:58.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:58.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:58.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:58.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-03-24T02:37:58.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:37:58.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:37:58.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:37:58.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:37:58.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-03-24T02:37:58.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_6.status 2026-03-24T02:37:58.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:37:58.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:58.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-03-24T02:37:58.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:58.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:37:58.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:37:58.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:37:58.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942]]], state=connected] 2026-03-24T02:37:58.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=4, tag_tid=1, entry_tid=7944], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942], [object_number=5, tag_tid=1, entry_tid=7941]]], state=connected] 2026-03-24T02:37:58.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:37:58.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=4, tag_tid=1, entry_tid=7944' 2026-03-24T02:37:58.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=4, tag_tid=1, entry_tid=7944' 2026-03-24T02:37:58.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=4, tag_tid=1, entry_tid=7944' '!=' 'object_number=4, tag_tid=1, entry_tid=7944' 2026-03-24T02:37:58.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:37:58.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:38:00.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_6 2026-03-24T02:38:00.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:00.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:00.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:38:00.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:00.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_6 ']' 2026-03-24T02:38:00.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-03-24T02:38:00.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-03-24T02:38:00.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:00.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:00.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:00.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:00.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:00.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:00.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:00.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:00.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-03-24T02:38:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:00.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:00.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:00.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:00.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:00.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:00.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:00.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:00.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:00.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:00.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:00.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:00.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:00.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:00.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:00.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:00.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:00.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:00.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:00.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:00.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-03-24T02:38:00.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-03-24T02:38:00.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:00.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:00.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:00.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:00.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:00.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:00.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:00.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:00.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:00.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-03-24T02:38:01.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-03-24T02:38:01.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:01.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:01.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_6 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:01.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:01.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:01.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:01.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:01.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:01.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:01.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:01.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:01.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:01.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:01.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:01.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:01.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_6 2026-03-24T02:38:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-03-24T02:38:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:38:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-03-24T02:38:01.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:01.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:01.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:38:01.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:01.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-03-24T02:38:01.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_6.status 2026-03-24T02:38:01.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:01.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:38:01.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-03-24T02:38:01.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:38:01.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:01.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:01.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:01.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942]]], state=connected] 2026-03-24T02:38:01.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942]]], state=connected] 2026-03-24T02:38:01.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:38:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-03-24T02:38:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:38:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-03-24T02:38:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:38:01.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:01.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-03-24T02:38:01.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_6.status 2026-03-24T02:38:01.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:01.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:38:01.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-03-24T02:38:01.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:38:01.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:01.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:01.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:01.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942]]], state=connected] 2026-03-24T02:38:01.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7943], [object_number=6, tag_tid=1, entry_tid=7942]]], state=connected] 2026-03-24T02:38:01.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.status 2026-03-24T02:38:01.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:01.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:01.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:01.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_6 snap 2026-03-24T02:38:01.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:01.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:01.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:38:01.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:38:01.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:38:01.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:38:01.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:38:02.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_6@snap 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_6': 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10d195ac81a 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10d195ac81a 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:26:22 2026 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:26:22 2026 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:27:23 2026 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10d195ac81a 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 42fe267d-df26-4b09-afa4-2668d198ad4a 2026-03-24T02:38:02.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:38:02.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:02.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_6 snap 2026-03-24T02:38:02.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:02.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:38:02.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:38:02.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:38:02.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.export 2026-03-24T02:38:02.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_6.export 2026-03-24T02:38:02.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_6.export 2026-03-24T02:38:02.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_6@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.export 2026-03-24T02:38:02.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:38:02.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_6@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_6.export 2026-03-24T02:38:02.833 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:38:02.838 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:38:02.829+0000 7fc159ee0640 -1 reset not still connected to 0x56419126a5b0 2026-03-24T02:38:02.841 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:38:02.833+0000 7fc159ee0640 -1 reset not still connected to 0x56419cc2d1e0 2026-03-24T02:38:02.841 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:38:02.833+0000 7fc159ee0640 -1 reset not still connected to 0x5641a270dd40 2026-03-24T02:38:02.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_6.export 2026-03-24T02:38:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_6.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_6.export 2026-03-24T02:38:02.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:02.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:38:02.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_7 2026-03-24T02:38:02.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_7 snap 2026-03-24T02:38:02.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:38:02.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:02.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:38:02.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:38:02.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_7@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:38:04.008 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:38:04.001+0000 7fb2796c9640 -1 reset not still connected to 0x55ac47237790 2026-03-24T02:38:04.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_7 2026-03-24T02:38:04.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:04.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:04.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:38:04.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_7 started 2026-03-24T02:38:04.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:04.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:04.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:38:04.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:38:04.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:38:04.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:04.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:38:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_7 started 2026-03-24T02:38:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:38:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:38:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:38:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:38:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_7 2026-03-24T02:38:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:38:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:38:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:38:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:38:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:05.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:38:05.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:05.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:38:05.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:05.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_7 2026-03-24T02:38:05.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:38:05.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:05.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:05.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:05.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:05.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:05.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:05.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:05.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:05.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:05.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:05.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:38:05.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:05.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:05.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:05.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:05.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_7 2026-03-24T02:38:05.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:05.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:05.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:05.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:05.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_7 2026-03-24T02:38:05.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:38:05.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:05.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:05.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:05.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:05.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:05.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:38:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:38:05.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:05.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:05.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:05.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:05.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_7 2026-03-24T02:38:05.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:05.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:05.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_7 2026-03-24T02:38:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:38:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:38:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:38:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:38:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:38:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:05.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:05.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:05.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:05.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_7 2026-03-24T02:38:05.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:05.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:05.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:05.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:05.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_7 2026-03-24T02:38:05.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:38:05.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:05.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:05.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:38:05.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:38:05.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:05.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:05.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:05.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:05.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:05.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:05.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:05.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:38:05.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:05.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:38:05.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:38:05.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:05.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:05.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:38:05.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:38:05.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:05.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:05.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:05.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:05.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:05.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:05.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:05.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:38:05.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:38:05.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:38:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_7 2026-03-24T02:38:05.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:38:05.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:38:05.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:38:05.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:38:05.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_7 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_7 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:05.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:38:05.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_7 2026-03-24T02:38:05.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:05.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:05.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:38:05.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:05.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_7 ']' 2026-03-24T02:38:05.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-03-24T02:38:05.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-03-24T02:38:05.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:05.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:05.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:05.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:05.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:05.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:05.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:05.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:05.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:05.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:05.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:05.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:05.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:05.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:05.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:05.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:05.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:05.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:05.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:05.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:05.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:05.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:05.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:05.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:05.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:06.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:06.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:06.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:06.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:06.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:06.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:06.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:06.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:06.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:06.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_7 2026-03-24T02:38:06.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:38:06.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:06.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:06.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:06.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:38:06.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:06.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:06.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:06.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:38:06.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:06.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:06.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:06.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:38:06.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:06.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:06.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:38:06.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:06.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:06.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:06.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:06.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:06.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:06.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:06.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:06.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:06.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:06.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:06.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:06.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:06.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:06.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:06.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:06.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:06.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:06.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:06.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:06.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_7 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:38:06.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:38:06.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:06.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:06.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:38:06.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:38:06.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:06.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:06.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:06.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:06.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:06.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:06.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:06.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:38:06.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:06.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:38:06.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:06.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:06.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:06.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:06.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:06.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-03-24T02:38:06.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:06.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:06.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:06.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-03-24T02:38:06.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:06.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:06.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:06.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:06.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-03-24T02:38:06.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_7.status 2026-03-24T02:38:06.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:06.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:06.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-03-24T02:38:06.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:06.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:06.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:06.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:06.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=8870], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:06.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=8870], [object_number=9, tag_tid=1, entry_tid=8869], [object_number=8, tag_tid=1, entry_tid=8868], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:06.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:06.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:06.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-03-24T02:38:06.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:06.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:06.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:06.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-03-24T02:38:06.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:06.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:06.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:06.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:06.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-03-24T02:38:06.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_7.status 2026-03-24T02:38:06.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-03-24T02:38:06.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:06.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:06.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:06.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=8870], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:06.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=8870], [object_number=9, tag_tid=1, entry_tid=8869], [object_number=8, tag_tid=1, entry_tid=8868], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:06.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:06.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=8870' 2026-03-24T02:38:06.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=8870' 2026-03-24T02:38:06.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=8870' '!=' '' 2026-03-24T02:38:06.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:38:06.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=8870' = '' 2026-03-24T02:38:06.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=10, tag_tid=1, entry_tid=8870' 2026-03-24T02:38:06.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:06.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:38:06.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:38:06.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=2 2026-03-24T02:38:06.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=10, tag_tid=1, entry_tid=8870' 2026-03-24T02:38:06.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:38:06.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:38:06.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=1 2026-03-24T02:38:06.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:06.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:38:06.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:38:06.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:38:06.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=10, tag_tid=1, entry_tid=8870' 2026-03-24T02:38:06.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:38:06.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:38:06.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=8870 2026-03-24T02:38:06.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 2 = 1 -a 1 -le 8870 2026-03-24T02:38:06.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:38:06.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:06.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:38:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_7 2026-03-24T02:38:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:38:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_7 ']' 2026-03-24T02:38:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-03-24T02:38:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-03-24T02:38:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:06.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:06.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:06.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:06.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:06.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:06.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:06.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:06.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:06.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:06.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:06.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:06.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-03-24T02:38:06.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:06.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:06.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:06.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:06.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:06.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:06.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:06.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:06.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:06.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:06.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:06.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:06.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:06.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:06.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:06.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:06.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:06.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:06.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:06.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_7 2026-03-24T02:38:06.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:38:06.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:06.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:06.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:06.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:38:06.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:06.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:06.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:06.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:38:06.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:06.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:06.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:06.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:38:06.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:06.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:06.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:38:06.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:06.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:06.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:06.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:06.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:06.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:06.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:06.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:06.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:06.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:06.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:06.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:06.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:06.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:06.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:06.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:06.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:06.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:07.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:07.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:07.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:07.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:07.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_7 2026-03-24T02:38:07.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:38:07.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:07.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:07.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:38:07.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:38:07.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:07.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:07.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:07.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:38:07.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:07.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:07.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:07.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:38:07.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:07.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:38:07.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:38:07.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:07.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:07.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:38:07.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:38:07.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:07.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:07.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:07.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:07.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:07.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:07.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:07.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:38:07.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:07.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:38:07.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:07.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:07.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:07.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:07.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:07.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-03-24T02:38:07.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:07.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:07.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:07.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-03-24T02:38:07.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:07.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:07.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:07.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:07.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-03-24T02:38:07.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_7.status 2026-03-24T02:38:07.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:07.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:07.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-03-24T02:38:07.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:07.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:07.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:07.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:07.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=8870], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:07.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=8870], [object_number=9, tag_tid=1, entry_tid=8869], [object_number=8, tag_tid=1, entry_tid=8868], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:07.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:07.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:07.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-03-24T02:38:07.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:07.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:07.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:07.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-03-24T02:38:07.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:07.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:07.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:07.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:07.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-03-24T02:38:07.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_7.status 2026-03-24T02:38:07.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:07.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:07.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-03-24T02:38:07.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:07.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:07.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:07.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:07.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=8870], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:07.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=8870], [object_number=9, tag_tid=1, entry_tid=8869], [object_number=8, tag_tid=1, entry_tid=8868], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:07.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:07.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=8870' 2026-03-24T02:38:07.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=8870' 2026-03-24T02:38:07.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=8870' '!=' 'object_number=10, tag_tid=1, entry_tid=8870' 2026-03-24T02:38:07.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:07.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:38:07.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_7 2026-03-24T02:38:07.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:07.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:07.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:38:07.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:07.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_7 ']' 2026-03-24T02:38:07.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-03-24T02:38:07.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-03-24T02:38:07.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:07.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:07.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:07.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:07.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:07.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:07.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:07.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:07.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:07.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:07.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:07.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:07.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:07.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:07.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:07.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:07.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:07.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:07.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:07.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-03-24T02:38:07.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:07.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:07.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:07.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:07.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:07.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:07.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:07.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:07.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:07.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:07.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:07.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:07.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_7 2026-03-24T02:38:07.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:38:07.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:07.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:07.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:07.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:38:07.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:07.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:07.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:07.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:38:07.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:07.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:07.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:07.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:38:07.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:07.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:07.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:38:07.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:07.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:07.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:07.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:07.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:07.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:07.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:07.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:07.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:07.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:07.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:07.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:07.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:07.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:07.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:07.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:07.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:07.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:07.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:07.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:07.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:07.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_7 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:38:07.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:07.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:38:07.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:07.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:07.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:07.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:07.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:07.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-03-24T02:38:07.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:07.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:07.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:07.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-03-24T02:38:07.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:07.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:07.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:07.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:07.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-03-24T02:38:07.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_7.status 2026-03-24T02:38:07.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:07.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:07.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-03-24T02:38:07.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:08.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:08.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:08.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:08.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=8870], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:08.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=8870], [object_number=9, tag_tid=1, entry_tid=8869], [object_number=8, tag_tid=1, entry_tid=8868], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:08.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:08.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:08.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-03-24T02:38:08.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:08.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:08.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:08.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-03-24T02:38:08.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:08.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:08.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:08.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:08.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-03-24T02:38:08.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_7.status 2026-03-24T02:38:08.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:08.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:08.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-03-24T02:38:08.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:08.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:08.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:08.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:08.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=8870], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:08.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=8870], [object_number=9, tag_tid=1, entry_tid=8869], [object_number=8, tag_tid=1, entry_tid=8868], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:08.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:08.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=8870' 2026-03-24T02:38:08.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=8870' 2026-03-24T02:38:08.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=8870' '!=' 'object_number=10, tag_tid=1, entry_tid=8870' 2026-03-24T02:38:08.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:08.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:38:08.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_7 2026-03-24T02:38:08.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:08.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:08.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:38:08.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:08.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_7 ']' 2026-03-24T02:38:08.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-03-24T02:38:08.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-03-24T02:38:08.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:08.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:08.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:08.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:08.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:08.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:08.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:08.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:08.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:08.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:08.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-03-24T02:38:08.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:08.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:08.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:08.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:08.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:08.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:08.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:08.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:08.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:08.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:08.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:08.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:08.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:08.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:08.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:08.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:08.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:08.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:08.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:08.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:08.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:08.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:08.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:08.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-03-24T02:38:08.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:08.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:08.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:08.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:08.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:08.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:08.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:08.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:08.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:08.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:08.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:08.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:08.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:08.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:08.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:08.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:08.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:08.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:08.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:08.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:08.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:08.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:08.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:08.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:08.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:08.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:08.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:08.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:08.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:08.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:08.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:08.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:08.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:08.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:09.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:09.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:09.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:09.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:09.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_7 2026-03-24T02:38:09.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:09.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:09.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:09.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:09.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:09.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:09.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:09.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:09.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:09.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:09.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_7 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:09.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:09.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:09.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:38:09.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:09.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:38:09.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:09.150 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:09.150 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:09.150 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:09.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:09.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-03-24T02:38:09.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:09.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:09.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:09.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-03-24T02:38:09.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:09.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:09.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:09.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:09.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-03-24T02:38:09.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_7.status 2026-03-24T02:38:09.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:09.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:09.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-03-24T02:38:09.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:09.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:09.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:09.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:09.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=8870], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:09.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=8870], [object_number=9, tag_tid=1, entry_tid=8869], [object_number=8, tag_tid=1, entry_tid=8868], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:09.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:09.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:09.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-03-24T02:38:09.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:09.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:09.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:09.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-03-24T02:38:09.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:09.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:09.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:09.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:09.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-03-24T02:38:09.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_7.status 2026-03-24T02:38:09.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:09.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:09.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-03-24T02:38:09.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:09.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:09.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:09.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:09.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=8870], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:09.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=8870], [object_number=9, tag_tid=1, entry_tid=8869], [object_number=8, tag_tid=1, entry_tid=8868], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:09.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:09.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=8870' 2026-03-24T02:38:09.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=8870' 2026-03-24T02:38:09.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=8870' '!=' 'object_number=10, tag_tid=1, entry_tid=8870' 2026-03-24T02:38:09.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:09.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:38:10.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_7 2026-03-24T02:38:10.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:10.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:10.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:38:10.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:10.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_7 ']' 2026-03-24T02:38:10.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-03-24T02:38:10.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-03-24T02:38:10.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:10.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:10.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:10.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:10.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:10.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:10.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:10.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:10.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:10.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:10.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-03-24T02:38:10.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:10.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:10.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:10.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:10.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:10.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:10.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:10.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:10.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:10.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:10.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:10.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:10.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:10.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:10.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:10.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:10.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:10.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:10.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:10.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:10.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:10.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:10.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:10.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:10.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:10.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:10.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:10.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:10.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:10.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:10.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:10.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:10.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:10.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:11.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:11.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:11.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:11.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:11.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-03-24T02:38:11.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:11.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:11.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:11.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:11.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:11.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:11.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:11.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:11.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:11.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:11.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:11.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:11.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:11.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:11.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:11.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:11.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:11.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:11.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:11.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:11.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:11.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:11.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:11.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:11.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:11.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:11.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:11.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:11.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:11.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:11.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:11.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:11.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:11.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:11.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:11.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:11.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:11.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:11.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_7 2026-03-24T02:38:11.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:38:11.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:11.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:11.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:11.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:38:11.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:11.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:11.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:11.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:38:11.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:11.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:11.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:11.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:38:11.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:11.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:11.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:38:11.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:11.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:11.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:11.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:11.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:11.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:11.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:11.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:11.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:11.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:11.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:11.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:11.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:11.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:11.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:11.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:11.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:11.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:11.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:11.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:11.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:11.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:11.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_7 2026-03-24T02:38:11.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:38:11.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:11.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:11.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:38:11.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:38:11.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:11.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:11.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:11.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:38:11.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:11.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:11.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:11.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:38:11.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:11.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:38:11.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:38:11.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:11.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:11.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:38:11.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:38:11.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:11.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:11.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:11.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:11.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:11.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:11.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:11.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:38:11.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:11.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:38:11.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:11.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:11.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:11.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:11.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:11.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-03-24T02:38:11.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:11.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:11.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:11.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-03-24T02:38:11.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:11.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:11.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:11.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:11.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-03-24T02:38:11.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_7.status 2026-03-24T02:38:11.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:11.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:11.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-03-24T02:38:11.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:11.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:11.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:11.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:11.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=8870], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:11.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=8870], [object_number=9, tag_tid=1, entry_tid=8869], [object_number=8, tag_tid=1, entry_tid=8868], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:11.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:11.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:11.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-03-24T02:38:11.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:11.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:11.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:11.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-03-24T02:38:11.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:11.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:11.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:11.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:11.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-03-24T02:38:11.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:11.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_7.status 2026-03-24T02:38:11.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:11.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-03-24T02:38:11.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=8870], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=8870], [object_number=9, tag_tid=1, entry_tid=8869], [object_number=8, tag_tid=1, entry_tid=8868], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:11.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:11.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=8870' 2026-03-24T02:38:11.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=8870' 2026-03-24T02:38:11.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=8870' '!=' 'object_number=10, tag_tid=1, entry_tid=8870' 2026-03-24T02:38:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:38:13.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_7 2026-03-24T02:38:13.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:13.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:13.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:38:13.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:13.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_7 ']' 2026-03-24T02:38:13.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-03-24T02:38:13.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-03-24T02:38:13.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:13.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:13.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:13.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:13.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:13.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:13.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:13.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:13.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-03-24T02:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:13.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:13.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:13.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:13.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:13.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:13.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:13.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:13.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:13.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:13.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:13.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:13.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:13.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:13.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:13.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:13.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:13.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:13.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:13.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:13.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:13.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:13.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:13.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:13.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:13.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:13.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:13.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_7 2026-03-24T02:38:13.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:38:13.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:13.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:13.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:13.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:38:13.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:13.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:13.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:13.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:38:13.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:13.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:13.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:13.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:38:13.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:38:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:13.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:13.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:13.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:13.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:13.832 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-03-24T02:38:13.832 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:13.832 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:13.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:13.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_7 2026-03-24T02:38:13.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:38:13.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:13.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:13.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:38:13.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:38:13.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:13.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:13.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:13.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:38:13.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:13.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:13.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:13.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:38:13.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:13.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:38:13.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:38:13.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:13.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:13.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:38:13.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:38:13.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:13.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:13.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:13.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:13.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:13.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:13.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:13.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:38:13.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:13.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:38:13.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:13.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:13.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:13.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_7 2026-03-24T02:38:13.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:13.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-03-24T02:38:13.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:13.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:13.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:13.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-03-24T02:38:13.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:13.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:13.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:13.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:13.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-03-24T02:38:13.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_7.status 2026-03-24T02:38:13.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:13.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:13.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-03-24T02:38:13.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:13.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:13.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:13.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:13.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=8870], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:13.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=8870], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:13.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:13.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:13.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-03-24T02:38:13.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:13.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:13.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:13.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-03-24T02:38:13.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:13.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:13.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:38:13.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:13.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-03-24T02:38:13.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_7.status 2026-03-24T02:38:13.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:13.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:13.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-03-24T02:38:13.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:13.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:13.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:13.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:13.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=8870], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:13.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=8870], [object_number=11, tag_tid=1, entry_tid=8867]]], state=connected] 2026-03-24T02:38:13.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.status 2026-03-24T02:38:13.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:13.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:13.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:13.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_7 snap 2026-03-24T02:38:13.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:13.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:13.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:38:13.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:38:13.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:38:13.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:38:13.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:38:14.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_7@snap 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_7': 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 116d3367bfb6 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.116d3367bfb6 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:26:30 2026 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:26:30 2026 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:27:31 2026 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 116d3367bfb6 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: b29736eb-7a53-4f8f-87b4-169199056317 2026-03-24T02:38:15.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:38:15.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:15.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_7 snap 2026-03-24T02:38:15.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:15.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:38:15.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:38:15.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:38:15.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.export 2026-03-24T02:38:15.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_7.export 2026-03-24T02:38:15.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_7.export 2026-03-24T02:38:15.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_7@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.export 2026-03-24T02:38:15.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:38:15.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_7@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_7.export 2026-03-24T02:38:15.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:38:15.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_7.export 2026-03-24T02:38:15.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_7.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_7.export 2026-03-24T02:38:15.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:15.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:38:15.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_8 2026-03-24T02:38:15.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_8 snap 2026-03-24T02:38:15.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:38:15.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:15.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:38:15.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:38:15.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_8@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:38:16.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_8 2026-03-24T02:38:16.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:16.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:16.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:38:16.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_8 started 2026-03-24T02:38:16.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:16.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:16.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:38:16.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:38:16.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:38:16.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:16.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:38:17.138 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:38:17.129+0000 7f5496ce4640 -1 reset not still connected to 0x5652cfec31e0 2026-03-24T02:38:17.138 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:38:17.133+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1e2ab60 2026-03-24T02:38:17.138 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:38:17.133+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1e2bc70 2026-03-24T02:38:17.148 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:38:17.141+0000 7f5496ce4640 -1 reset not still connected to 0x5652ebdff2b0 2026-03-24T02:38:17.244 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:38:17.237+0000 7fc159ee0640 -1 reset not still connected to 0x5641a57e6c30 2026-03-24T02:38:17.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_8 started 2026-03-24T02:38:17.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:17.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:17.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:38:17.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:38:17.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:38:17.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:38:17.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_8 2026-03-24T02:38:17.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:38:17.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:38:17.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:17.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:17.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:38:17.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:38:17.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:17.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:38:17.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:17.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:38:17.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:17.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_8 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:38:17.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:38:17.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:17.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:17.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:17.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:17.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_8 2026-03-24T02:38:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-03-24T02:38:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:17.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:17.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_8 2026-03-24T02:38:17.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:38:17.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:17.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:17.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:17.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:17.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:17.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:17.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:17.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:17.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:38:17.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:17.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:17.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:17.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:17.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_8 2026-03-24T02:38:17.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:38:17.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:38:17.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:38:17.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_8 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_8 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:38:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:38:17.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:17.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:38:18.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_8 2026-03-24T02:38:18.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:18.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:18.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:38:18.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:18.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_8 ']' 2026-03-24T02:38:18.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-03-24T02:38:18.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-03-24T02:38:18.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:18.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:18.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:18.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:18.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:18.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:18.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:18.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:18.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:18.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:18.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-03-24T02:38:18.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:18.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:18.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:18.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:18.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:18.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:18.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:18.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-03-24T02:38:18.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-03-24T02:38:18.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:18.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:18.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:18.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:18.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:18.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:18.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:18.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:18.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:18.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:18.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:18.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:18.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:18.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2026-03-24T02:38:18.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:18.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-03-24T02:38:18.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:18.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:18.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:18.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-03-24T02:38:18.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:18.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:18.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:18.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:18.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-03-24T02:38:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_8.status 2026-03-24T02:38:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:18.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-03-24T02:38:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:18.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:18.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:18.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:18.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:18.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:18.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:18.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:18.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-03-24T02:38:18.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:18.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:18.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:18.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-03-24T02:38:18.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:18.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:18.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:18.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:18.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-03-24T02:38:18.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_8.status 2026-03-24T02:38:18.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-03-24T02:38:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:18.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:18.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:18.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:18.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:18.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:18.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' '' 2026-03-24T02:38:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:38:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' = '' 2026-03-24T02:38:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:38:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:38:18.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:38:18.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:38:18.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:18.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:38:18.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:38:18.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:18.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:38:18.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:38:18.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:38:18.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:18.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:38:18.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:38:18.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=1 2026-03-24T02:38:18.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 1 2026-03-24T02:38:18.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:38:18.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:18.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:38:18.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_8 2026-03-24T02:38:18.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:18.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:18.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:38:18.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:18.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_8 ']' 2026-03-24T02:38:18.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-03-24T02:38:18.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-03-24T02:38:18.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:18.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:18.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:18.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:18.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:18.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:18.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:18.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:18.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:18.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:18.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:18.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:18.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:18.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:18.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:18.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-03-24T02:38:18.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-03-24T02:38:18.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:18.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:18.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:18.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:18.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:18.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:18.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:18.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:18.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:18.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:18.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:18.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:18.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:18.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:18.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:18.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:18.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:18.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2026-03-24T02:38:18.753 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:38:18.745+0000 7fb2796c9640 -1 reset not still connected to 0x55ac32b66a90 2026-03-24T02:38:18.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:18.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-03-24T02:38:18.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:18.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:18.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:18.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-03-24T02:38:18.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:18.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:18.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:18.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:18.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-03-24T02:38:18.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_8.status 2026-03-24T02:38:18.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:18.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:18.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-03-24T02:38:18.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:18.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:18.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:18.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:18.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:18.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:18.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:18.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:18.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-03-24T02:38:18.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:18.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:18.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:18.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-03-24T02:38:18.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:18.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:18.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:18.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:18.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-03-24T02:38:18.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_8.status 2026-03-24T02:38:18.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:18.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:18.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-03-24T02:38:18.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:18.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:18.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:18.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:18.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:18.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:18.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:18.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:18.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:18.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:18.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:18.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:38:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_8 2026-03-24T02:38:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:38:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_8 ']' 2026-03-24T02:38:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-03-24T02:38:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-03-24T02:38:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:19.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:19.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:19.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:19.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-03-24T02:38:19.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:19.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:19.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:19.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:19.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:19.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:19.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:19.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:19.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:19.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:19.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:19.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:19.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:19.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:19.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:19.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:19.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:19.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-03-24T02:38:19.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-03-24T02:38:19.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:19.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:19.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:19.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2026-03-24T02:38:19.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:19.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:19.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:19.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:19.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:19.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:19.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:19.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:19.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:19.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:19.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:19.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:19.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:19.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:19.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:19.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:19.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:19.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:19.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:19.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:19.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:19.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:19.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:19.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:19.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:19.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:19.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:19.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:19.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:19.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:19.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:19.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:19.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:19.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2026-03-24T02:38:19.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:19.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-03-24T02:38:19.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:19.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:19.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:19.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-03-24T02:38:19.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:19.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:19.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:19.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:19.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-03-24T02:38:19.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_8.status 2026-03-24T02:38:19.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:19.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:19.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-03-24T02:38:19.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:19.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:19.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:19.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:19.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:19.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:19.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:19.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:19.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-03-24T02:38:19.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:19.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:19.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:19.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-03-24T02:38:19.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:19.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:19.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:19.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:19.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-03-24T02:38:19.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_8.status 2026-03-24T02:38:19.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:19.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:19.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-03-24T02:38:19.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:19.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:19.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:19.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:19.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:19.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:19.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:19.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:19.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:19.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:19.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:19.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:38:20.198 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:38:20.193+0000 7fc159ee0640 -1 reset not still connected to 0x56417efc0dd0 2026-03-24T02:38:20.203 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:38:20.197+0000 7fc159ee0640 -1 reset not still connected to 0x56419d60f790 2026-03-24T02:38:20.203 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:38:20.197+0000 7fc159ee0640 -1 reset not still connected to 0x5641a57e6c30 2026-03-24T02:38:20.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_8 2026-03-24T02:38:20.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:20.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:20.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:38:20.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:20.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_8 ']' 2026-03-24T02:38:20.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-03-24T02:38:20.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-03-24T02:38:20.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:20.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:20.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:20.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:20.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:20.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:20.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:20.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:20.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-03-24T02:38:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:20.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:20.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:20.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:20.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:20.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-03-24T02:38:20.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-03-24T02:38:20.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:20.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2026-03-24T02:38:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:20.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:20.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:20.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:20.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2026-03-24T02:38:20.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:20.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-03-24T02:38:20.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:20.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:20.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:20.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-03-24T02:38:20.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:20.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:20.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:20.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:20.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-03-24T02:38:20.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_8.status 2026-03-24T02:38:20.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:20.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:20.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-03-24T02:38:20.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:20.546 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-03-24T02:38:20.546 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-03-24T02:38:20.546 INFO:tasks.rbd_mirror_thrash:waiting for 8.226985790346966 secs before reviving daemons 2026-03-24T02:38:20.547 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:38:20.545+0000 7f4288303640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 52155) UID: 1000 2026-03-24T02:38:20.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:20.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:20.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:20.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:20.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:20.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:20.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:20.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-03-24T02:38:20.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:20.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:20.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:20.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-03-24T02:38:20.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:20.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:20.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:20.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:20.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-03-24T02:38:20.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_8.status 2026-03-24T02:38:20.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:20.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:20.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-03-24T02:38:20.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:20.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:20.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:20.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:20.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:20.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:20.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:20.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:20.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:20.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:20.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:20.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:38:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_8 2026-03-24T02:38:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:38:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_8 ']' 2026-03-24T02:38:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-03-24T02:38:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-03-24T02:38:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:22.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:22.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:22.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:22.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:22.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:22.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:22.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:22.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:22.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:22.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:22.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:22.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-03-24T02:38:22.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-03-24T02:38:22.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:22.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2026-03-24T02:38:22.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:22.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:22.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:22.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:22.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:22.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:22.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:22.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:22.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:22.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:22.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:22.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2026-03-24T02:38:22.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:22.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-03-24T02:38:22.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:22.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:22.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:22.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-03-24T02:38:22.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:22.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:22.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:22.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:22.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-03-24T02:38:22.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_8.status 2026-03-24T02:38:22.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:22.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:22.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-03-24T02:38:22.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:22.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:22.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:22.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:22.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:22.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:22.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:22.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:22.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-03-24T02:38:22.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:22.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:22.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:22.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-03-24T02:38:22.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:22.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:22.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:22.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:22.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-03-24T02:38:22.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_8.status 2026-03-24T02:38:22.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:22.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:22.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-03-24T02:38:22.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:22.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:22.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:22.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:22.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:22.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:22.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:22.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:22.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:22.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:22.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:22.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:38:22.809 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:38:22.801+0000 7f5496ce4640 -1 reset not still connected to 0x5652c92ea5b0 2026-03-24T02:38:22.810 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:38:22.801+0000 7f5496ce4640 -1 reset not still connected to 0x5652d714f930 2026-03-24T02:38:22.810 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:38:22.801+0000 7f5496ce4640 -1 reset not still connected to 0x5652ef236ea0 2026-03-24T02:38:24.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_8 2026-03-24T02:38:24.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:24.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:24.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:38:24.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:24.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_8 ']' 2026-03-24T02:38:24.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-03-24T02:38:24.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-03-24T02:38:24.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:24.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:24.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:24.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:24.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:24.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:24.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:24.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:24.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-03-24T02:38:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:24.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:24.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:24.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:24.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:24.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:24.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:24.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:24.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:24.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:24.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:24.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:24.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:24.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:24.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:24.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:24.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:24.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:24.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:24.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:24.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-03-24T02:38:24.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-03-24T02:38:24.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:24.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2026-03-24T02:38:24.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:24.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:24.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:24.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:24.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:24.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:24.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:24.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:24.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:24.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:24.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:24.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:24.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:24.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:24.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:24.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:24.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:24.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:24.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:24.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:24.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:24.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:24.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:24.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:24.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:24.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:24.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:24.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:24.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:24.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:24.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:24.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:24.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:24.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2026-03-24T02:38:24.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:24.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-03-24T02:38:24.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:24.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:24.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:24.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-03-24T02:38:24.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:24.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:24.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:24.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:24.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-03-24T02:38:24.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_8.status 2026-03-24T02:38:24.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:24.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:24.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-03-24T02:38:24.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:24.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:24.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:24.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:24.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:24.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:24.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:24.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:24.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-03-24T02:38:24.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:24.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:24.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:24.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-03-24T02:38:24.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:24.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:24.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:24.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:24.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-03-24T02:38:24.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_8.status 2026-03-24T02:38:24.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:24.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:24.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-03-24T02:38:24.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:24.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:24.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:24.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:24.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:24.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:24.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:24.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:24.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:24.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:24.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:24.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:38:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_8 2026-03-24T02:38:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:38:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_8 ']' 2026-03-24T02:38:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-03-24T02:38:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-03-24T02:38:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:26.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:26.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:26.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:26.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:26.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:26.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:26.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:26.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:26.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:26.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:26.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:26.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:26.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:26.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-03-24T02:38:26.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-03-24T02:38:26.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:26.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2026-03-24T02:38:26.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:26.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:26.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:26.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:26.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:26.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:26.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:26.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:26.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:26.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:26.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:26.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:26.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:26.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:26.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:26.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:26.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:26.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2026-03-24T02:38:26.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:26.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-03-24T02:38:26.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:26.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:26.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:26.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-03-24T02:38:26.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:26.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:26.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:26.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:26.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-03-24T02:38:26.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_8.status 2026-03-24T02:38:26.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-03-24T02:38:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:26.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:26.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:26.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:26.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:26.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:26.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:26.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:26.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-03-24T02:38:26.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:26.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:26.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:26.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-03-24T02:38:26.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:26.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:26.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:38:26.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:26.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-03-24T02:38:26.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_8.status 2026-03-24T02:38:26.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:26.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:26.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-03-24T02:38:26.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:27.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:27.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:27.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:27.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:27.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4891], [object_number=6, tag_tid=1, entry_tid=4890]]], state=connected] 2026-03-24T02:38:27.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.status 2026-03-24T02:38:27.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:27.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:27.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:27.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_8 snap 2026-03-24T02:38:27.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:27.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:27.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:38:27.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:38:27.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:38:27.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:38:27.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:38:28.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_8@snap 2026-03-24T02:38:28.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_8': 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 119ed0c5425d 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.119ed0c5425d 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:26:44 2026 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:26:44 2026 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:30:55 2026 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 119ed0c5425d 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 58466e3c-9b77-4bde-ae34-22dde0d466a5 2026-03-24T02:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:38:28.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:28.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_8 snap 2026-03-24T02:38:28.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:28.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:38:28.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:38:28.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:38:28.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.export 2026-03-24T02:38:28.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_8.export 2026-03-24T02:38:28.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_8.export 2026-03-24T02:38:28.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_8@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.export 2026-03-24T02:38:28.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:38:28.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_8@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_8.export 2026-03-24T02:38:28.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:38:28.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_8.export 2026-03-24T02:38:28.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_8.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_8.export 2026-03-24T02:38:28.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:28.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:38:28.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_9 2026-03-24T02:38:28.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_9 snap 2026-03-24T02:38:28.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:38:28.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:28.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:38:28.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:38:28.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_9@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:38:28.774 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-03-24T02:38:28.774 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:38:28.774 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-03-24T02:38:28.774 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-03-24T02:38:28.774 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-03-24T02:38:28.775 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-03-24T02:38:28.775 INFO:tasks.rbd_mirror_thrash:waiting for 91.46465027225108 secs before thrashing 2026-03-24T02:38:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_9 2026-03-24T02:38:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:38:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_9 started 2026-03-24T02:38:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:38:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:38:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:38:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:38:31.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_9 started 2026-03-24T02:38:31.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:31.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:31.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:38:31.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:38:31.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:38:31.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:38:31.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_9 2026-03-24T02:38:31.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:38:31.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:38:31.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:31.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:31.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:38:31.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:38:31.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:31.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:38:31.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:31.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:38:31.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:31.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_9 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:38:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:38:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:31.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:31.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:31.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:31.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_9 2026-03-24T02:38:32.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-03-24T02:38:32.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:32.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:32.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:32.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_9 2026-03-24T02:38:32.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:38:32.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:32.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:32.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:32.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:32.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:38:32.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:32.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:32.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:32.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:32.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_9 2026-03-24T02:38:32.095 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-03-24T02:38:32.095 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:32.095 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:32.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:32.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_9 2026-03-24T02:38:32.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:38:32.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:32.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:32.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:32.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:32.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:32.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:32.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:32.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:32.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:32.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:32.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:32.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:32.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:32.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:32.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:38:32.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:32.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:32.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:32.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:38:32.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:32.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:32.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:32.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:32.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:32.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:32.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:32.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:38:32.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:38:32.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:32.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:32.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:32.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:32.113 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_9 2026-03-24T02:38:32.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:38:32.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:38:32.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:38:32.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:38:32.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_9 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_9 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:32.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:38:32.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_9 2026-03-24T02:38:32.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:32.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:32.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:38:32.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:32.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_9 ']' 2026-03-24T02:38:32.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-03-24T02:38:32.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-03-24T02:38:32.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:32.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:32.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:32.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:32.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:32.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:32.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:32.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:32.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:32.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:32.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:32.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:32.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:32.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:32.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:32.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:32.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:32.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-03-24T02:38:32.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:32.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:32.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:32.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:32.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:32.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:32.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:32.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:32.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:32.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:32.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:32.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:32.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:32.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:32.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:32.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:32.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-03-24T02:38:32.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:32.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:32.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:32.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_9 2026-03-24T02:38:32.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:38:32.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:32.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:32.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:32.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:38:32.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:32.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:32.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:32.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:38:32.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:32.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:32.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:32.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:38:32.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:32.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:32.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:38:32.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:32.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:32.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:32.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:32.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:32.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:32.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:32.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:32.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:32.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:32.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:32.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:32.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:32.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:32.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:32.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:32.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:32.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:32.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:32.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-03-24T02:38:32.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:32.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:32.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:32.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-03-24T02:38:32.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:32.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:32.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:32.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:32.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-03-24T02:38:32.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_9.status 2026-03-24T02:38:32.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:32.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:32.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-03-24T02:38:32.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:32.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:32.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:32.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:32.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:32.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10689], [object_number=8, tag_tid=1, entry_tid=10688], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:32.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:32.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:32.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-03-24T02:38:32.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:32.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:32.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:32.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-03-24T02:38:32.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:32.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:32.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:32.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:32.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-03-24T02:38:32.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_9.status 2026-03-24T02:38:32.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:32.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:32.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-03-24T02:38:32.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:32.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:32.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:32.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:32.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:32.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10689], [object_number=8, tag_tid=1, entry_tid=10688], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:32.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=10689' '!=' '' 2026-03-24T02:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=10689' = '' 2026-03-24T02:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:38:32.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=2 2026-03-24T02:38:32.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:32.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:38:32.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:38:32.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=1 2026-03-24T02:38:32.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:32.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:38:32.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:38:32.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:38:32.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:32.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:38:32.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:38:32.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=10689 2026-03-24T02:38:32.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 2 = 1 -a 1 -le 10689 2026-03-24T02:38:32.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:38:32.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:32.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:38:33.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_9 2026-03-24T02:38:33.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:33.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:33.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:38:33.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:33.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_9 ']' 2026-03-24T02:38:33.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-03-24T02:38:33.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-03-24T02:38:33.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:33.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:33.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:33.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:33.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:33.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:33.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:33.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:33.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:33.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:33.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-03-24T02:38:33.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:33.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:33.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:33.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:33.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:33.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:33.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:33.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:33.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:33.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:33.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:33.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:33.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:33.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:33.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:33.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:33.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:33.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:33.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:33.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:33.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:33.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:33.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:33.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:33.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:33.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:33.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:33.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:33.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:33.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:33.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:33.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:33.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:33.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:33.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-03-24T02:38:33.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:33.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:33.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:33.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-03-24T02:38:33.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:33.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:33.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:33.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:33.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:33.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:33.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:33.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:33.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:33.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:33.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:33.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:33.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:33.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:33.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:33.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:33.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:33.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:33.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-03-24T02:38:33.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:33.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:33.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:33.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_9 2026-03-24T02:38:33.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:38:33.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:33.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:33.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:33.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:38:33.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:33.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:33.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:33.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:38:33.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:33.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:33.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:33.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:38:33.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:33.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:33.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:38:33.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:33.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:33.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:33.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:33.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:33.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:33.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:33.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:33.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:33.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:33.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:33.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:33.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:33.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:33.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:33.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:33.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:33.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:33.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:33.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-03-24T02:38:33.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:33.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:33.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:33.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-03-24T02:38:33.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:33.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:33.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:33.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:33.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-03-24T02:38:33.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_9.status 2026-03-24T02:38:33.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:33.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:33.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-03-24T02:38:33.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:33.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:33.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:33.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:33.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:33.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10689], [object_number=8, tag_tid=1, entry_tid=10688], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:33.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-03-24T02:38:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-03-24T02:38:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:33.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-03-24T02:38:33.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_9.status 2026-03-24T02:38:33.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:33.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-03-24T02:38:33.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10689], [object_number=8, tag_tid=1, entry_tid=10688], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:33.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:33.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:33.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:33.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=10689' '!=' 'object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:33.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:33.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:38:33.839 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:38:33.833+0000 7fc159ee0640 -1 reset not still connected to 0x5641a57e6c30 2026-03-24T02:38:33.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_9 2026-03-24T02:38:33.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:33.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:33.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:38:33.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:33.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_9 ']' 2026-03-24T02:38:33.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-03-24T02:38:33.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-03-24T02:38:33.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:33.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:33.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:33.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:33.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:33.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:33.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:33.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:33.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:33.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:33.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:33.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:33.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:33.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:33.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:33.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:34.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-03-24T02:38:34.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:34.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:34.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:34.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-03-24T02:38:34.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:34.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:34.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:34.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:34.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:34.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:34.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:34.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:34.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:34.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:34.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:34.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:34.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:34.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:34.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:34.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:34.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:34.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:34.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:34.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:34.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:34.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:34.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:34.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:34.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:34.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:34.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:34.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:34.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:34.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:34.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:34.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:34.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:34.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:34.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-03-24T02:38:34.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:34.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:34.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:34.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_9 2026-03-24T02:38:34.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:38:34.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:34.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:34.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:34.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:38:34.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:34.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:34.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:34.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:38:34.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:34.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:34.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:34.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:38:34.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:34.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:34.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:38:34.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:34.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:34.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:34.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:34.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:34.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:34.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:34.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:34.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:34.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:34.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:34.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:34.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:34.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:34.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:34.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:34.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:34.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:34.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-03-24T02:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-03-24T02:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-03-24T02:38:34.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_9.status 2026-03-24T02:38:34.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:34.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-03-24T02:38:34.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:34.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:34.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:34.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:34.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:34.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10689], [object_number=8, tag_tid=1, entry_tid=10688], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:34.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:34.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-03-24T02:38:34.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:34.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:34.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:34.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-03-24T02:38:34.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:34.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:34.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:34.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:34.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-03-24T02:38:34.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_9.status 2026-03-24T02:38:34.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:34.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:34.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-03-24T02:38:34.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:34.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:34.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:34.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:34.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:34.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10689], [object_number=8, tag_tid=1, entry_tid=10688], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:34.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:34.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:34.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:34.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=10689' '!=' 'object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:34.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:34.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:38:34.326 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:38:34.321+0000 7fc159ee0640 -1 reset not still connected to 0x56417cdd40d0 2026-03-24T02:38:34.329 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:38:34.321+0000 7fc159ee0640 -1 reset not still connected to 0x5641874ab520 2026-03-24T02:38:34.348 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:38:34.341+0000 7fc159ee0640 -1 reset not still connected to 0x56419adcd860 2026-03-24T02:38:34.348 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:38:34.341+0000 7fc159ee0640 -1 reset not still connected to 0x5641a27bf1e0 2026-03-24T02:38:35.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_9 2026-03-24T02:38:35.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:35.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:35.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:38:35.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:35.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_9 ']' 2026-03-24T02:38:35.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-03-24T02:38:35.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-03-24T02:38:35.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:35.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:35.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:35.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:35.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:35.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:35.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:35.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:35.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:35.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:35.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-03-24T02:38:35.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:35.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:35.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:35.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:35.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:35.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:35.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:35.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:35.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:35.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:35.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:35.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:35.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:35.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:35.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:35.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:35.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:35.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:35.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:35.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:35.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:35.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:35.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:35.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:35.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:35.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:35.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:35.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:35.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:35.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:35.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:35.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:35.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:35.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:35.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-03-24T02:38:35.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:35.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-03-24T02:38:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:35.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:35.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:35.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:35.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:35.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-03-24T02:38:35.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:35.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:35.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:35.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_9 2026-03-24T02:38:35.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:38:35.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:35.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:35.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:35.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:35.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:35.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:35.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:35.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:35.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:35.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-03-24T02:38:35.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:35.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:35.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:35.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-03-24T02:38:35.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:35.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:35.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:35.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:35.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-03-24T02:38:35.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_9.status 2026-03-24T02:38:35.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:35.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:35.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-03-24T02:38:35.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:35.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:35.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:35.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:35.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:35.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10689], [object_number=8, tag_tid=1, entry_tid=10688], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:35.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:35.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:35.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-03-24T02:38:35.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:35.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:35.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:35.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-03-24T02:38:35.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:35.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:35.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:35.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:35.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-03-24T02:38:35.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_9.status 2026-03-24T02:38:35.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:35.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:35.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-03-24T02:38:35.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:35.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:35.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:35.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:35.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:35.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10689], [object_number=8, tag_tid=1, entry_tid=10688], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:35.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:35.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:35.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:35.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=10689' '!=' 'object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:35.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:35.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:38:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_9 2026-03-24T02:38:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:38:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_9 ']' 2026-03-24T02:38:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-03-24T02:38:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-03-24T02:38:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:37.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-03-24T02:38:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:37.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:37.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:37.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:37.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:37.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:37.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:37.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:37.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:37.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:37.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:37.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:37.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:37.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:37.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-03-24T02:38:37.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:37.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:37.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-03-24T02:38:37.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:37.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:37.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:37.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_9 2026-03-24T02:38:37.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:38:37.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:37.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:37.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:37.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:38:37.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:37.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:37.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:37.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:38:37.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:37.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:37.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:37.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:38:37.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:37.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:37.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:38:37.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:37.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:37.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:37.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:37.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:37.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:37.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:37.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:37.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:37.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:37.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:37.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:37.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:37.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:37.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:37.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:37.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:37.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-03-24T02:38:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-03-24T02:38:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:37.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-03-24T02:38:37.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_9.status 2026-03-24T02:38:37.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:37.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:37.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-03-24T02:38:37.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:37.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:37.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:37.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:37.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:37.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10689], [object_number=8, tag_tid=1, entry_tid=10688], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:37.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:37.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:37.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-03-24T02:38:37.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:37.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:37.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:37.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-03-24T02:38:37.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:37.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:37.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:37.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:37.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-03-24T02:38:37.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_9.status 2026-03-24T02:38:37.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:37.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:37.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-03-24T02:38:37.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:37.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:37.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:37.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:37.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:37.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10689], [object_number=8, tag_tid=1, entry_tid=10688], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:37.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:37.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:37.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:37.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=10689' '!=' 'object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:37.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:37.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:38:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_9 2026-03-24T02:38:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:38:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_9 ']' 2026-03-24T02:38:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-03-24T02:38:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-03-24T02:38:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:39.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:39.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:39.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:39.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:39.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:39.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:39.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:39.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:39.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:39.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:39.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:39.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:39.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:39.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:39.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:39.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:39.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:39.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:39.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:39.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:39.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-03-24T02:38:39.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:39.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:39.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:39.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:39.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:39.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:39.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:39.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:39.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:39.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:39.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:39.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:39.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:39.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:39.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:39.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:39.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:39.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:39.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:39.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:39.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-03-24T02:38:39.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:39.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_9 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:39.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:39.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:39.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:39.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:39.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:39.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:39.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:39.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:39.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:39.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:39.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:39.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-03-24T02:38:39.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:39.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:39.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:39.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-03-24T02:38:39.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:39.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:39.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:39.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:39.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-03-24T02:38:39.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_9.status 2026-03-24T02:38:39.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:39.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:39.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-03-24T02:38:39.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:39.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:39.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:39.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:39.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:39.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10689], [object_number=8, tag_tid=1, entry_tid=10688], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:39.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:39.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:39.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-03-24T02:38:39.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:39.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:39.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:39.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-03-24T02:38:39.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:39.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:39.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:39.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:39.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-03-24T02:38:39.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_9.status 2026-03-24T02:38:39.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:39.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:39.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-03-24T02:38:39.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10689], [object_number=8, tag_tid=1, entry_tid=10688], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=10689' '!=' 'object_number=9, tag_tid=1, entry_tid=10689' 2026-03-24T02:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:38:41.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_9 2026-03-24T02:38:41.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:41.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:41.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:38:41.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:41.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_9 ']' 2026-03-24T02:38:41.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-03-24T02:38:41.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-03-24T02:38:41.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:41.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:41.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:41.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:41.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:41.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:41.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:41.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:41.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:41.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:41.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:41.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:41.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:41.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:41.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:41.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:41.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:41.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:41.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:41.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:41.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:41.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:41.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:42.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-03-24T02:38:42.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:42.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:42.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:42.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:42.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:42.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:42.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:42.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:42.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:42.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:42.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:42.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:42.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:42.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:42.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:42.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:42.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:42.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:42.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:42.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-03-24T02:38:42.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:42.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_9 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:42.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:42.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:42.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:42.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:42.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_9 2026-03-24T02:38:42.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:42.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-03-24T02:38:42.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:42.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:42.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:42.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-03-24T02:38:42.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:42.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:42.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:42.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:42.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-03-24T02:38:42.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_9.status 2026-03-24T02:38:42.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:42.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:42.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-03-24T02:38:42.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:42.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:42.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:42.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:42.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:42.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:42.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:42.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:42.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-03-24T02:38:42.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:42.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:42.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:42.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-03-24T02:38:42.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:42.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:42.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:38:42.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:42.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-03-24T02:38:42.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_9.status 2026-03-24T02:38:42.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:42.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:42.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-03-24T02:38:42.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:42.286 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:38:42.286 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:38:42.286 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:42.286 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:42.286 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10687], [object_number=10, tag_tid=1, entry_tid=10686]]], state=connected] 2026-03-24T02:38:42.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.status 2026-03-24T02:38:42.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:42.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2026-03-24T02:38:42.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:42.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_9 snap 2026-03-24T02:38:42.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:42.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:42.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:38:42.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:38:42.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:38:42.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:38:42.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:38:43.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_9@snap 2026-03-24T02:38:43.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_9': 2026-03-24T02:38:43.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:38:43.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:38:43.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:38:43.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c9169c0d57 2026-03-24T02:38:43.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c9169c0d57 2026-03-24T02:38:43.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:38:43.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:38:43.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:38:43.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:38:43.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:26:50 2026 2026-03-24T02:38:43.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:26:50 2026 2026-03-24T02:38:43.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:27:51 2026 2026-03-24T02:38:43.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:38:43.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c9169c0d57 2026-03-24T02:38:43.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:38:43.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:38:43.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 0947aed1-66d1-4268-b1e7-51427f503356 2026-03-24T02:38:43.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:38:43.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:43.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_9 snap 2026-03-24T02:38:43.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:43.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:38:43.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:38:43.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:38:43.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.export 2026-03-24T02:38:43.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_9.export 2026-03-24T02:38:43.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_9.export 2026-03-24T02:38:43.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_9@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.export 2026-03-24T02:38:43.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:38:43.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_9@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_9.export 2026-03-24T02:38:43.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:38:43.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_9.export 2026-03-24T02:38:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_9.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_9.export 2026-03-24T02:38:43.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:43.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:38:43.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_10 2026-03-24T02:38:43.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_10 snap 2026-03-24T02:38:43.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:38:43.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:43.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:38:43.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:38:43.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_10@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:38:44.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_10 2026-03-24T02:38:44.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:44.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:44.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:38:44.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_10 started 2026-03-24T02:38:44.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:44.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:44.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:38:44.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:38:44.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:38:44.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:44.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:38:45.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_10 started 2026-03-24T02:38:45.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:45.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:45.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:38:45.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:38:45.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:38:45.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:38:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:38:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_10 2026-03-24T02:38:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:38:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:38:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:38:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:38:45.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:45.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:38:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_10 2026-03-24T02:38:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:38:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:38:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:38:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:38:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:38:45.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:45.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:45.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:45.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:45.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_10 2026-03-24T02:38:45.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-03-24T02:38:45.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:45.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:45.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:45.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_10 2026-03-24T02:38:45.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:38:45.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:45.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:45.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:45.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:45.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:45.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:45.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:45.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:45.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:45.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:45.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:45.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:45.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:45.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:45.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:38:45.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:45.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:45.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:45.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:38:45.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:45.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:45.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:45.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:45.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:45.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:45.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:45.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:38:45.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:38:45.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:45.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:45.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:45.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:45.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_10 2026-03-24T02:38:45.434 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:38:45.429+0000 7fc159ee0640 -1 reset not still connected to 0x5641a57e6c30 2026-03-24T02:38:45.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-03-24T02:38:45.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:45.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:45.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:45.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_10 2026-03-24T02:38:45.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:38:45.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:45.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:45.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:45.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:45.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:45.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:45.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:45.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:45.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:45.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:45.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:45.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:45.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:45.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:45.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:38:45.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:45.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:45.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:45.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:38:45.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:45.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:45.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:45.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:45.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:45.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:45.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:45.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:38:45.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:38:45.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:45.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:45.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:45.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:45.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_10 2026-03-24T02:38:45.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-03-24T02:38:45.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:45.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_10 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:38:45.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:38:45.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:45.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:45.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:45.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:45.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:45.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:45.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:45.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:38:45.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:38:45.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:38:45.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:45.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:45.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:45.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_10 2026-03-24T02:38:45.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-03-24T02:38:45.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:45.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:45.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:45.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 1 2026-03-24T02:38:46.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:46.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:38:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_10 2026-03-24T02:38:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:38:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:38:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:38:46.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:46.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:46.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:46.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:46.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_10 2026-03-24T02:38:46.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-03-24T02:38:46.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:46.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:46.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:46.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_10 2026-03-24T02:38:46.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:38:46.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:46.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:46.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:46.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:46.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:46.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:46.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:46.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:46.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:46.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:46.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:46.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:46.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:46.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:46.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:38:46.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:46.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:46.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:46.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:38:46.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:46.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:46.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:46.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:46.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:46.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:46.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:46.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:38:46.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:38:46.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:46.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:46.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:46.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:46.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_10 2026-03-24T02:38:46.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-03-24T02:38:46.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:46.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:46.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:46.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_10 2026-03-24T02:38:46.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:38:46.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:46.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:46.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:46.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:46.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:46.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:46.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:46.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:46.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:46.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:46.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:46.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:46.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:46.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:46.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:38:46.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:46.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:46.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:46.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:38:46.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:46.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:46.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:46.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:46.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:46.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:46.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:46.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:38:46.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:38:46.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:46.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:46.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:46.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:46.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_10 2026-03-24T02:38:46.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-03-24T02:38:46.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:46.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:46.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:46.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_10 2026-03-24T02:38:46.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:38:46.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:46.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:46.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:38:46.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:38:46.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:46.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:46.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:46.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:46.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:46.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:46.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:46.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:38:46.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:46.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:38:46.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:38:46.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:46.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:46.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:38:46.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:38:46.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:46.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:46.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:46.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:46.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:46.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:46.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:46.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:38:46.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:38:46.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:38:46.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:46.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:46.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:46.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_10 2026-03-24T02:38:46.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-03-24T02:38:46.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:46.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:46.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:46.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 2 2026-03-24T02:38:48.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:48.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:38:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_10 2026-03-24T02:38:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:38:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:48.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:48.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:48.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:38:48.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:48.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:48.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:48.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:38:48.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:48.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:48.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:48.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:48.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:48.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:48.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:48.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:38:48.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:38:48.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:48.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:48.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:48.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:48.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_10 2026-03-24T02:38:49.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-03-24T02:38:49.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:49.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:49.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:49.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_10 2026-03-24T02:38:49.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:38:49.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:49.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:49.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:49.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:49.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:49.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:49.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:49.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:49.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:49.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:49.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:49.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:49.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:49.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:49.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:38:49.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:49.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:49.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:49.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:38:49.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:49.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:49.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:49.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:49.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:49.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:49.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:49.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:38:49.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:38:49.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:49.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:49.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:49.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:49.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_10 2026-03-24T02:38:49.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-03-24T02:38:49.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:49.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:49.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:49.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_10 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:38:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:38:49.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:49.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:49.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:49.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:49.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_10 2026-03-24T02:38:49.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-03-24T02:38:49.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:49.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:49.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:49.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_10 2026-03-24T02:38:49.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:38:49.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:49.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:49.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:38:49.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:38:49.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:49.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:38:49.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:38:49.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:49.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:49.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:49.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_10 2026-03-24T02:38:49.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-03-24T02:38:49.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:49.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:49.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:49.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 4 2026-03-24T02:38:49.559 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:38:49.553+0000 7fc159ee0640 -1 reset not still connected to 0x56417efc0dd0 2026-03-24T02:38:49.565 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:38:49.557+0000 7fc159ee0640 -1 reset not still connected to 0x5641a57e6c30 2026-03-24T02:38:53.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:53.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_10 2026-03-24T02:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:53.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:53.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:38:53.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:53.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:53.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:53.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:38:53.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:53.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:53.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:53.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:53.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:53.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:53.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:53.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:38:53.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:38:53.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:53.320 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:53.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:53.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:53.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_10 2026-03-24T02:38:53.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:38:53.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:38:53.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:38:53.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:38:53.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_10 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_10 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:53.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:38:53.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_10 2026-03-24T02:38:53.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:53.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:53.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:38:53.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:53.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_10 ']' 2026-03-24T02:38:53.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_10' 2026-03-24T02:38:53.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_10 2026-03-24T02:38:53.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:53.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:53.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:53.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:53.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:53.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:53.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:53.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:53.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_10 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:53.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:53.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:53.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:53.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:53.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:53.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:53.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_10 2026-03-24T02:38:53.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_10 2026-03-24T02:38:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_10 2026-03-24T02:38:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_10 '' 2026-03-24T02:38:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_10 2026-03-24T02:38:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-03-24T02:38:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_10.status 2026-03-24T02:38:53.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:53.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_10.status 2026-03-24T02:38:53.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-03-24T02:38:53.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_10.status 2026-03-24T02:38:53.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:53.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:53.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:53.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=4678], [object_number=7, tag_tid=1, entry_tid=4675]]], state=connected] 2026-03-24T02:38:53.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=4678], [object_number=7, tag_tid=1, entry_tid=4675]]], state=connected] 2026-03-24T02:38:53.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_10.status 2026-03-24T02:38:53.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:53.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_10 2026-03-24T02:38:53.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:53.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:53.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_10 2026-03-24T02:38:53.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_10 '..*' 2026-03-24T02:38:53.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:53.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:53.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_10 2026-03-24T02:38:53.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:53.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-03-24T02:38:53.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_10.status 2026-03-24T02:38:53.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:53.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_10.status 2026-03-24T02:38:53.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-03-24T02:38:53.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_10.status 2026-03-24T02:38:53.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:53.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:53.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:53.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=4678], [object_number=7, tag_tid=1, entry_tid=4675]]], state=connected] 2026-03-24T02:38:53.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=4678], [object_number=7, tag_tid=1, entry_tid=4675]]], state=connected] 2026-03-24T02:38:53.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_10.status 2026-03-24T02:38:53.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:53.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:53.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:53.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_10 snap 2026-03-24T02:38:53.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:53.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:53.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:38:53.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:38:53.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:38:53.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:38:53.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:38:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_10@snap 2026-03-24T02:38:54.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_10': 2026-03-24T02:38:54.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:38:54.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:38:54.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:38:54.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10d1878973ec 2026-03-24T02:38:54.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10d1878973ec 2026-03-24T02:38:54.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:38:54.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:38:54.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:38:54.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:38:54.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:27:05 2026 2026-03-24T02:38:54.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:27:05 2026 2026-03-24T02:38:54.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:28:47 2026 2026-03-24T02:38:54.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:38:54.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10d1878973ec 2026-03-24T02:38:54.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:38:54.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:38:54.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: af1b9448-880e-41c9-b13d-fd3d514c9e21 2026-03-24T02:38:54.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:38:54.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:54.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_10 snap 2026-03-24T02:38:54.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:54.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:38:54.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:38:54.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:38:54.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_10.export 2026-03-24T02:38:54.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_10.export 2026-03-24T02:38:54.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_10.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_10.export 2026-03-24T02:38:54.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_10@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_10.export 2026-03-24T02:38:55.095 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:38:55.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_10@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_10.export 2026-03-24T02:38:55.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:38:55.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_10.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_10.export 2026-03-24T02:38:55.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_10.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_10.export 2026-03-24T02:38:55.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:55.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:38:55.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_11 2026-03-24T02:38:55.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_11 snap 2026-03-24T02:38:55.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:38:55.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:55.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:38:55.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:38:55.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_11@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:38:56.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_11 2026-03-24T02:38:56.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:56.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:56.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:38:56.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_11 started 2026-03-24T02:38:56.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:56.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:56.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:38:56.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:38:56.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:38:56.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:56.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:38:57.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_11 started 2026-03-24T02:38:57.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:57.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:57.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:38:57.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:38:57.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:38:57.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:38:57.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_11 2026-03-24T02:38:57.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:38:57.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:38:57.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:57.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:57.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:38:57.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:38:57.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:57.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:38:57.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:57.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:57.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:57.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:38:57.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:38:57.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:57.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:57.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:57.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:57.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11 2026-03-24T02:38:57.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:38:57.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:57.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_11 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:38:57.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:57.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:57.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:57.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:38:57.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:57.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:57.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:57.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:57.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:57.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:57.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:57.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:38:57.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:38:57.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:57.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:57.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:57.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:57.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_11 2026-03-24T02:38:57.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:38:57.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:57.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:57.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:57.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_11 2026-03-24T02:38:57.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:38:57.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:57.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:57.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:57.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:57.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:38:57.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:38:57.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:57.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:57.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:57.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:57.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_11 2026-03-24T02:38:57.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:38:57.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:38:57.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:57.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:57.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_11 2026-03-24T02:38:57.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:38:57.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:38:57.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:57.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:38:57.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:38:57.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:38:57.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:57.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:57.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:57.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:57.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:38:57.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:57.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:38:57.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:57.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:38:57.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:38:57.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:38:57.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:38:57.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:38:57.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:38:57.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:38:57.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:38:57.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:38:57.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:57.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:57.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:38:57.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:38:57.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:38:57.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:38:57.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:38:57.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:57.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:57.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:57.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_11 2026-03-24T02:38:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:38:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:38:57.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:38:57.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_11 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_11 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:38:57.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_11 2026-03-24T02:38:57.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:57.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:57.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:38:57.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:57.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_11 ']' 2026-03-24T02:38:57.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-03-24T02:38:57.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-03-24T02:38:57.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:57.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:57.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:57.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:57.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:57.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:57.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-03-24T02:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:57.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:57.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:57.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:57.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:57.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-03-24T02:38:57.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:38:57.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:57.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:57.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:57.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-03-24T02:38:57.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:57.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:57.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:57.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:57.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:57.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:57.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:57.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:57.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:57.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:57.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:57.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:57.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:57.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:57.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:57.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:57.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:57.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:57.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:57.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:57.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:57.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:57.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:57.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:57.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:57.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:57.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:57.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:57.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:57.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:57.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:57.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:57.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:57.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-03-24T02:38:58.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:38:58.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:58.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:58.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:58.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_11 2026-03-24T02:38:58.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:38:58.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:58.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:58.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:58.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:58.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:58.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:58.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:58.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:58.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:58.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:58.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:58.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:58.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:58.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:58.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:58.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_11 2026-03-24T02:38:58.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:38:58.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:58.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_11 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:38:58.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:58.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:38:58.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_11 2026-03-24T02:38:58.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:58.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-03-24T02:38:58.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:58.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:58.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:38:58.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-03-24T02:38:58.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:58.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:58.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:38:58.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:58.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-03-24T02:38:58.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_11.status 2026-03-24T02:38:58.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:58.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:58.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-03-24T02:38:58.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:58.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:58.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:58.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:58.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:38:58.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:38:58.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:58.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:58.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-03-24T02:38:58.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:58.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:58.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:38:58.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-03-24T02:38:58.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:58.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:58.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:38:58.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:58.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-03-24T02:38:58.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_11.status 2026-03-24T02:38:58.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-03-24T02:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:58.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:58.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:58.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:58.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:38:58.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:38:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' '' 2026-03-24T02:38:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:38:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' = '' 2026-03-24T02:38:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:38:58.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:38:58.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:38:58.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:58.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:38:58.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:38:58.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:38:58.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:58.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:38:58.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:38:58.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:38:58.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:58.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:38:58.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:38:58.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=1 2026-03-24T02:38:58.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 1 2026-03-24T02:38:58.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:38:58.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:58.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:38:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_11 2026-03-24T02:38:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:38:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_11 ']' 2026-03-24T02:38:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-03-24T02:38:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-03-24T02:38:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-03-24T02:38:58.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:38:58.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:58.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:58.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:58.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-03-24T02:38:58.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:58.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:58.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:58.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:58.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:58.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-03-24T02:38:58.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:38:58.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:58.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:58.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:58.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_11 2026-03-24T02:38:58.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:38:58.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:58.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:58.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:58.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:38:58.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:58.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:58.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:58.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:38:58.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:58.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:58.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:58.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:38:58.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:58.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:58.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:38:58.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:58.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:58.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:58.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:58.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:58.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:58.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:58.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:58.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:58.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:58.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:58.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:58.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:58.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:58.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:58.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:58.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:58.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_11 2026-03-24T02:38:58.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:38:58.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:58.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:58.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:58.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_11 2026-03-24T02:38:58.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:38:58.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:58.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:58.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:38:58.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:38:58.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:58.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:58.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:58.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:38:58.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:58.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:58.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:58.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:38:58.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:58.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:38:58.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:38:58.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:58.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:58.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:38:58.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:38:58.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:58.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:58.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:58.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:58.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:58.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:58.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:58.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:38:58.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:58.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:38:58.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:58.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:58.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:58.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_11 2026-03-24T02:38:58.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:58.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-03-24T02:38:58.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:58.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:58.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:38:58.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-03-24T02:38:58.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:58.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:58.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:38:58.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:58.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-03-24T02:38:58.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_11.status 2026-03-24T02:38:58.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:58.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:58.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-03-24T02:38:58.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:58.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:58.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:58.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:58.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:38:58.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:38:58.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:58.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:58.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-03-24T02:38:58.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:58.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:58.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:38:58.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-03-24T02:38:58.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:58.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:58.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:38:58.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:58.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-03-24T02:38:58.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_11.status 2026-03-24T02:38:58.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:58.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:58.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-03-24T02:38:58.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:58.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:58.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:58.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:58.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:38:58.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:38:58.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:58.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:58.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:58.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:58.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:58.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:38:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_11 2026-03-24T02:38:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:38:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:38:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:38:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_11 ']' 2026-03-24T02:38:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-03-24T02:38:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-03-24T02:38:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:38:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:38:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:38:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:38:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:38:59.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:38:59.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:38:59.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:59.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:59.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:38:59.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:38:59.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:59.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:59.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:38:59.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:38:59.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:38:59.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:59.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:59.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:38:59.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-03-24T02:38:59.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:38:59.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:59.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:38:59.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:38:59.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:59.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:59.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:38:59.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-03-24T02:38:59.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:38:59.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:59.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:59.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_11 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:59.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:38:59.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:38:59.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:38:59.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:59.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:59.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:38:59.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_11 2026-03-24T02:38:59.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:38:59.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:38:59.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:38:59.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:38:59.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_11 2026-03-24T02:38:59.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:38:59.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:38:59.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:38:59.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:38:59.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:59.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:59.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:38:59.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_11 2026-03-24T02:38:59.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:38:59.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-03-24T02:38:59.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:59.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:59.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:38:59.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-03-24T02:38:59.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:59.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:59.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:38:59.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:38:59.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-03-24T02:38:59.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_11.status 2026-03-24T02:38:59.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:59.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:59.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-03-24T02:38:59.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:59.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:59.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:59.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:59.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:38:59.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:38:59.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:59.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:38:59.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-03-24T02:38:59.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:59.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:59.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:38:59.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-03-24T02:38:59.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:38:59.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:38:59.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:38:59.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:38:59.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-03-24T02:38:59.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_11.status 2026-03-24T02:38:59.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:38:59.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:59.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-03-24T02:38:59.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:59.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:38:59.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:38:59.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:38:59.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:38:59.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:38:59.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:38:59.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:59.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:59.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:38:59.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:38:59.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:39:00.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_11 2026-03-24T02:39:00.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:00.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:00.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:39:00.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:00.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_11 ']' 2026-03-24T02:39:00.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-03-24T02:39:00.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-03-24T02:39:00.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:00.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:00.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:00.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:00.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:00.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:00.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:00.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:00.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:00.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:00.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:00.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:00.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:00.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:00.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:00.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:00.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:00.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:00.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:00.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:00.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:00.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:00.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:00.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-03-24T02:39:00.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:39:00.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:00.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:00.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:00.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:00.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:00.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:00.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:00.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:00.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:00.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:00.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:00.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:00.615 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-03-24T02:39:00.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:39:00.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:00.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:00.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:00.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_11 2026-03-24T02:39:00.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:00.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:00.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:00.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:00.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:00.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:00.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:00.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:00.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:00.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:00.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:00.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:00.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_11 2026-03-24T02:39:00.746 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:39:00.746 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:00.746 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:00.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:00.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_11 2026-03-24T02:39:00.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:39:00.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:00.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:00.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:39:00.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:39:00.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:00.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:00.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:00.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:39:00.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:00.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:00.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:00.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:39:00.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:39:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:39:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:39:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:39:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:39:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:39:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:39:00.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:00.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:00.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:00.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_11 2026-03-24T02:39:00.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:00.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-03-24T02:39:00.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:00.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:00.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:39:00.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-03-24T02:39:00.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:00.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:00.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:39:00.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:00.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-03-24T02:39:00.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_11.status 2026-03-24T02:39:00.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:00.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:00.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-03-24T02:39:00.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:00.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:00.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:00.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:00.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:39:00.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:39:00.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:00.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:00.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-03-24T02:39:00.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:00.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:00.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:39:00.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-03-24T02:39:00.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:00.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:00.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:39:00.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:00.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-03-24T02:39:00.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_11.status 2026-03-24T02:39:00.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:00.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:00.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-03-24T02:39:00.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:00.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:00.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:00.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:00.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:39:00.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:39:00.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:00.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:39:00.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:39:00.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:39:00.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:00.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:39:01.543 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:39:01.537+0000 7fb2796c9640 -1 reset not still connected to 0x55ac1d3a3ad0 2026-03-24T02:39:01.546 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:39:01.541+0000 7fb2796c9640 -1 reset not still connected to 0x55ac2004e000 2026-03-24T02:39:01.549 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:39:01.541+0000 7fb2796c9640 -1 reset not still connected to 0x55ac3b5661a0 2026-03-24T02:39:02.137 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:02.129+0000 7f5496ce4640 -1 reset not still connected to 0x5652c68aaa90 2026-03-24T02:39:02.137 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:02.129+0000 7f5496ce4640 -1 reset not still connected to 0x5652c77f9380 2026-03-24T02:39:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_11 2026-03-24T02:39:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:39:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_11 ']' 2026-03-24T02:39:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-03-24T02:39:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-03-24T02:39:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:02.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:02.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:02.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-03-24T02:39:02.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:02.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:02.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:02.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:02.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:02.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:02.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:02.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:02.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:02.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:02.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:02.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:02.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:02.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:02.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:02.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:02.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:02.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:02.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:02.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:02.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:02.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:02.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:02.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:02.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:02.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:02.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:02.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:02.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:02.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-03-24T02:39:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:39:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:02.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:02.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-03-24T02:39:02.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:02.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:02.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:02.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:02.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:02.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:02.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:02.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:02.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:02.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:02.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:02.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:02.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:02.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:02.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:02.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:02.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:02.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:02.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:02.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:02.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:02.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:02.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:02.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:02.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:02.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:02.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:02.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:02.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:02.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:02.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:02.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:02.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:02.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-03-24T02:39:02.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:39:02.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:02.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:02.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:02.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_11 2026-03-24T02:39:02.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:02.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:02.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:02.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:02.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:02.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:02.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:02.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:02.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:02.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:02.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:02.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:02.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:02.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:02.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:02.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_11 2026-03-24T02:39:02.753 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:39:02.753 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:02.753 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:02.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:02.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_11 2026-03-24T02:39:02.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:39:02.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:02.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:02.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:39:02.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:39:02.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:02.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:02.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:02.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:39:02.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:02.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:02.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:02.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:39:02.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:02.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:39:02.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:39:02.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:02.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:02.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:39:02.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:39:02.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:02.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:02.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:02.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:02.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:02.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:02.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:02.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:39:02.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:39:02.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:39:02.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:02.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:02.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:02.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_11 2026-03-24T02:39:02.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:02.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-03-24T02:39:02.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:02.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:02.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:39:02.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-03-24T02:39:02.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:02.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:02.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:39:02.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:02.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-03-24T02:39:02.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_11.status 2026-03-24T02:39:02.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:02.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:02.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:02.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-03-24T02:39:02.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:02.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:02.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:02.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:39:02.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:39:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:02.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:02.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-03-24T02:39:02.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:02.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:02.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:39:02.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-03-24T02:39:02.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:02.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:02.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:39:02.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:02.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-03-24T02:39:02.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_11.status 2026-03-24T02:39:02.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:02.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:02.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-03-24T02:39:02.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:02.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:02.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:02.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:02.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:39:02.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:39:02.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:02.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:39:02.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:39:02.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:39:02.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:02.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:39:04.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_11 2026-03-24T02:39:04.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:04.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:04.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:39:04.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:04.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_11 ']' 2026-03-24T02:39:04.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-03-24T02:39:04.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-03-24T02:39:04.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:04.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:04.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:04.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:04.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:04.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:04.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:04.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:04.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:04.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:04.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-03-24T02:39:04.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:04.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:04.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:04.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:04.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:04.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:04.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:04.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:04.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:04.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:04.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-03-24T02:39:05.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:39:05.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:05.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:05.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:05.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:05.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:05.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:05.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:05.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:05.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:05.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-03-24T02:39:05.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:39:05.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:05.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:05.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:05.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_11 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:05.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:05.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:05.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:05.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:05.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:05.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:05.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:05.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:05.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:05.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:05.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:05.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:05.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:05.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_11 2026-03-24T02:39:05.105 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:05.097+0000 7f5496ce4640 -1 reset not still connected to 0x5652d714f930 2026-03-24T02:39:05.111 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:05.105+0000 7f5496ce4640 -1 reset not still connected to 0x5652d7d67a00 2026-03-24T02:39:05.111 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:05.105+0000 7f5496ce4640 -1 reset not still connected to 0x5652ebdffba0 2026-03-24T02:39:05.111 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:05.105+0000 7f5496ce4640 -1 reset not still connected to 0x5652ef236ea0 2026-03-24T02:39:05.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:39:05.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:05.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_11 2026-03-24T02:39:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:39:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:39:05.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:05.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:05.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:05.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_11 2026-03-24T02:39:05.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:05.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-03-24T02:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-03-24T02:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-03-24T02:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_11.status 2026-03-24T02:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:05.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:05.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-03-24T02:39:05.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:05.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:05.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:05.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:05.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:39:05.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:39:05.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:05.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:05.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-03-24T02:39:05.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:05.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:05.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:39:05.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-03-24T02:39:05.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:05.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:05.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:39:05.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:05.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-03-24T02:39:05.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_11.status 2026-03-24T02:39:05.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:05.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:05.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-03-24T02:39:05.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:05.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:05.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:05.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:05.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:39:05.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:39:05.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:05.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:39:05.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:39:05.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' 'object_number=5, tag_tid=2, entry_tid=1' 2026-03-24T02:39:05.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:05.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:39:07.329 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:07.321+0000 7f5496ce4640 -1 reset not still connected to 0x5652c92ea5b0 2026-03-24T02:39:07.330 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:07.325+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c31520 2026-03-24T02:39:07.330 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:07.325+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c5de10 2026-03-24T02:39:07.332 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:07.325+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1e2bad0 2026-03-24T02:39:07.332 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:07.325+0000 7f5496ce4640 -1 reset not still connected to 0x5652d714f930 2026-03-24T02:39:07.332 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:07.325+0000 7f5496ce4640 -1 reset not still connected to 0x5652d7d67a00 2026-03-24T02:39:07.333 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:07.325+0000 7f5496ce4640 -1 reset not still connected to 0x5652ebdffba0 2026-03-24T02:39:07.333 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:07.325+0000 7f5496ce4640 -1 reset not still connected to 0x5652ef236ea0 2026-03-24T02:39:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_11 2026-03-24T02:39:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:39:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_11 ']' 2026-03-24T02:39:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-03-24T02:39:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-03-24T02:39:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:07.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:07.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:07.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:07.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-03-24T02:39:07.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:07.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:07.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:07.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:07.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:07.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:07.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:07.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:07.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:07.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:07.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:07.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:07.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-03-24T02:39:07.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:39:07.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:07.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:07.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:07.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-03-24T02:39:07.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:07.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:07.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:07.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:07.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:07.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:07.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:07.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:07.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:07.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:07.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:07.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:07.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:07.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:07.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:07.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:07.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:07.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:07.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:07.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:07.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:07.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:07.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:07.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:07.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:07.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:07.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:07.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:07.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:07.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:07.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:07.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:07.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:07.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-03-24T02:39:07.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:39:07.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:07.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:07.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:07.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_11 2026-03-24T02:39:07.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:07.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:07.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:07.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:07.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:07.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:07.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:07.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:07.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:07.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:07.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:07.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:07.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:07.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_11 2026-03-24T02:39:07.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-03-24T02:39:07.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:07.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:07.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:07.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_11 2026-03-24T02:39:07.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:39:07.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:39:07.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:39:07.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:07.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:07.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:07.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_11 2026-03-24T02:39:07.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:07.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-03-24T02:39:07.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:07.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:07.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:39:07.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-03-24T02:39:07.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:07.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:07.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:39:07.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:07.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-03-24T02:39:07.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_11.status 2026-03-24T02:39:07.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:07.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:07.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-03-24T02:39:07.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:07.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:07.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:07.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:07.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:39:07.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:39:07.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:07.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:07.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-03-24T02:39:07.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:07.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:07.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:39:07.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-03-24T02:39:07.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:07.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:07.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:39:07.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:07.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-03-24T02:39:07.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_11.status 2026-03-24T02:39:07.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:07.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:07.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-03-24T02:39:07.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:07.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:07.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:07.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:07.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:39:07.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=4607], [object_number=6, tag_tid=1, entry_tid=4606]]], state=connected] 2026-03-24T02:39:07.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.status 2026-03-24T02:39:07.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:07.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:07.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:07.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_11 snap 2026-03-24T02:39:07.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:07.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:07.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:39:07.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:07.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:39:07.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:39:07.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:39:08.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_11@snap 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_11': 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 116d8695e10d 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.116d8695e10d 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:27:12 2026 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:27:12 2026 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:28:13 2026 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 116d8695e10d 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: a029d6f0-f7cb-4e39-adc6-e3ec45e10c84 2026-03-24T02:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:39:08.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:08.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_11 snap 2026-03-24T02:39:08.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:08.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:39:08.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:08.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:39:08.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.export 2026-03-24T02:39:08.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_11.export 2026-03-24T02:39:08.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_11.export 2026-03-24T02:39:08.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_11@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.export 2026-03-24T02:39:09.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:39:09.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_11@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_11.export 2026-03-24T02:39:09.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:39:09.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_11.export 2026-03-24T02:39:09.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_11.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_11.export 2026-03-24T02:39:09.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:09.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:39:09.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_12 2026-03-24T02:39:09.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_12 snap 2026-03-24T02:39:09.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:09.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:09.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:39:09.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:09.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_12@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:39:10.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_12 2026-03-24T02:39:10.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:10.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:10.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:39:10.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_12 started 2026-03-24T02:39:10.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:10.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:10.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:39:10.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:39:10.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:39:10.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:10.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:39:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_12 started 2026-03-24T02:39:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:39:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:39:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:39:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:39:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:39:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_12 2026-03-24T02:39:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:39:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:39:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:39:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:39:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:39:11.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:11.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_12 2026-03-24T02:39:11.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:39:11.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:11.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:11.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:11.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:11.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:11.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:11.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:11.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:11.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:11.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:11.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:11.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:11.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:11.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:11.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:39:11.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:11.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:11.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:11.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:39:11.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:11.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:11.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:11.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:11.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:11.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:11.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:11.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:39:11.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:39:11.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:11.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:11.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:11.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:11.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_12 2026-03-24T02:39:11.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:39:11.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:39:11.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:39:11.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:39:11.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:39:11.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:39:11.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:39:11.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:11.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_12 2026-03-24T02:39:11.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:39:11.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:11.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:39:11.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:39:11.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:39:11.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:39:11.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_12 2026-03-24T02:39:11.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:39:11.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:11.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:39:11.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:39:11.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:39:11.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:39:11.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:39:11.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:39:11.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:11.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:39:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_12 2026-03-24T02:39:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:39:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_12 ']' 2026-03-24T02:39:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-03-24T02:39:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-03-24T02:39:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:11.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:11.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:11.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:11.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-03-24T02:39:11.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:11.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:11.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:11.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:11.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:11.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:11.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:11.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:11.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:11.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:11.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:11.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:11.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:11.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:11.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:11.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:11.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:11.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-03-24T02:39:11.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:11.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-03-24T02:39:11.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:11.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:11.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:11.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-03-24T02:39:11.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:11.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:11.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:11.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:11.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-03-24T02:39:11.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_12.status 2026-03-24T02:39:11.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:11.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:11.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-03-24T02:39:11.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:12.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:12.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:12.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:12.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:12.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:12.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:12.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=4, entry_tid=1' 2026-03-24T02:39:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-03-24T02:39:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-03-24T02:39:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-03-24T02:39:12.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_12.status 2026-03-24T02:39:12.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:12.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:12.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-03-24T02:39:12.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:12.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:12.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:12.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:12.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:12.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:12.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:12.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:12.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=4, entry_tid=1' -a 'object_number=1, tag_tid=4, entry_tid=1' = 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:12.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=1' '!=' '' 2026-03-24T02:39:12.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:39:12.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=1' = '' 2026-03-24T02:39:12.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:12.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=1, tag_tid=4, entry_tid=1' 2026-03-24T02:39:12.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:39:12.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=4 2026-03-24T02:39:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:39:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=3 2026-03-24T02:39:12.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=1, tag_tid=4, entry_tid=1' 2026-03-24T02:39:12.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:39:12.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:39:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:39:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=1 2026-03-24T02:39:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 4 = 3 -a 1 -le 1 2026-03-24T02:39:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:39:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:39:12.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_12 2026-03-24T02:39:12.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:12.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:12.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:39:12.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:12.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_12 ']' 2026-03-24T02:39:12.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-03-24T02:39:12.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-03-24T02:39:12.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:12.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:12.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:12.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:12.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:12.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:12.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:12.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:12.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:12.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:12.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-03-24T02:39:12.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:12.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:12.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:12.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:12.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:12.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:12.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:12.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:12.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:12.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:12.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:12.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:12.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:12.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:12.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:12.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:12.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:12.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:12.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:12.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:12.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:12.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:12.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:12.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:12.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:12.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:12.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:12.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:12.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:12.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:12.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:12.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:12.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:12.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-03-24T02:39:12.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:12.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-03-24T02:39:12.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:12.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:12.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:12.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-03-24T02:39:12.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:12.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:12.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:12.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:12.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-03-24T02:39:12.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_12.status 2026-03-24T02:39:12.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:12.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:12.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:12.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-03-24T02:39:12.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:12.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:12.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:12.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:12.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:12.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:12.403 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=4, entry_tid=1' 2026-03-24T02:39:12.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-03-24T02:39:12.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:12.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:12.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:12.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-03-24T02:39:12.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:12.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:12.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:12.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:12.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-03-24T02:39:12.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_12.status 2026-03-24T02:39:12.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:12.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:12.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-03-24T02:39:12.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:12.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:12.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:12.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:12.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:12.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:12.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:12.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:12.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=4, entry_tid=1' -a 'object_number=1, tag_tid=4, entry_tid=1' = 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:12.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=1' '!=' 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:12.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:12.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:39:12.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_12 2026-03-24T02:39:12.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:12.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:12.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:39:12.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:12.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_12 ']' 2026-03-24T02:39:12.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-03-24T02:39:12.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-03-24T02:39:12.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:12.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:12.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:12.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:12.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:12.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:12.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:12.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:12.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:12.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:12.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:12.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:12.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:12.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:12.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:12.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:12.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:12.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:12.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-03-24T02:39:12.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:12.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-03-24T02:39:12.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:12.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:12.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:12.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-03-24T02:39:12.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:12.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:12.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:12.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:12.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-03-24T02:39:12.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_12.status 2026-03-24T02:39:12.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:12.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:12.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-03-24T02:39:12.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:13.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:13.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:13.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:13.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:13.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:13.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:13.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=4, entry_tid=1' 2026-03-24T02:39:13.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-03-24T02:39:13.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:13.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:13.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:13.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-03-24T02:39:13.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:13.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:13.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:13.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:13.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-03-24T02:39:13.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_12.status 2026-03-24T02:39:13.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:13.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:13.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-03-24T02:39:13.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:13.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:13.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:13.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:13.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:13.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:13.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:13.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:13.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=4, entry_tid=1' -a 'object_number=1, tag_tid=4, entry_tid=1' = 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:13.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=1' '!=' 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:13.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:13.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:39:13.436 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:13.429+0000 7f5496ce4640 -1 reset not still connected to 0x5652cd369110 2026-03-24T02:39:13.436 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:13.429+0000 7f5496ce4640 -1 reset not still connected to 0x5652cd369380 2026-03-24T02:39:13.438 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:13.433+0000 7f5496ce4640 -1 reset not still connected to 0x5652cd3696c0 2026-03-24T02:39:13.438 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:13.433+0000 7f5496ce4640 -1 reset not still connected to 0x5652cfec2750 2026-03-24T02:39:13.438 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:13.433+0000 7f5496ce4640 -1 reset not still connected to 0x5652d14669c0 2026-03-24T02:39:13.438 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:13.433+0000 7f5496ce4640 -1 reset not still connected to 0x5652e4d709c0 2026-03-24T02:39:13.438 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:13.433+0000 7f5496ce4640 -1 reset not still connected to 0x5652ec2ad930 2026-03-24T02:39:13.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_12 2026-03-24T02:39:13.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:13.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:13.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:39:13.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:13.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_12 ']' 2026-03-24T02:39:13.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-03-24T02:39:13.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-03-24T02:39:13.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:13.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:13.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:13.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:13.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:13.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:13.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:13.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:13.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:13.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:13.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-03-24T02:39:13.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:13.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:13.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:13.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:13.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:13.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:13.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:13.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:13.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:13.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:13.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:13.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:13.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:13.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:13.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-03-24T02:39:13.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:13.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-03-24T02:39:13.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:13.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:13.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:13.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-03-24T02:39:13.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:13.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:13.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:13.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:13.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-03-24T02:39:13.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:13.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_12.status 2026-03-24T02:39:13.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:13.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-03-24T02:39:13.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:13.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:13.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:13.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:13.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:13.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:13.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:13.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=4, entry_tid=1' 2026-03-24T02:39:13.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-03-24T02:39:13.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:13.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:13.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:13.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-03-24T02:39:13.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:13.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:13.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:13.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:13.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-03-24T02:39:13.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_12.status 2026-03-24T02:39:13.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:13.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:13.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-03-24T02:39:13.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:14.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:14.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:14.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:14.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:14.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:14.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:14.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:14.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=4, entry_tid=1' -a 'object_number=1, tag_tid=4, entry_tid=1' = 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:14.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=1' '!=' 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:14.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:14.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:39:15.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_12 2026-03-24T02:39:15.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:15.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:15.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:39:15.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:15.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_12 ']' 2026-03-24T02:39:15.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-03-24T02:39:15.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-03-24T02:39:15.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:15.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:15.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:15.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:15.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:15.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:15.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:15.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:15.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:15.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:15.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:15.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:15.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:15.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:15.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:15.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-03-24T02:39:15.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:15.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-03-24T02:39:15.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:15.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:15.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:15.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-03-24T02:39:15.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:15.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:15.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:15.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:15.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-03-24T02:39:15.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_12.status 2026-03-24T02:39:15.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:15.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:15.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-03-24T02:39:15.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:15.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:15.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:15.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:15.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:15.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:15.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:15.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=4, entry_tid=1' 2026-03-24T02:39:15.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-03-24T02:39:15.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:15.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:15.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:15.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-03-24T02:39:15.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:15.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:15.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:15.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:15.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-03-24T02:39:15.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_12.status 2026-03-24T02:39:15.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:15.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:15.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-03-24T02:39:15.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:15.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:15.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:15.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:15.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:15.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:15.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:15.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:15.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=4, entry_tid=1' -a 'object_number=1, tag_tid=4, entry_tid=1' = 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:15.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=1' '!=' 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:15.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:15.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:39:17.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_12 2026-03-24T02:39:17.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:17.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:17.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:39:17.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:17.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_12 ']' 2026-03-24T02:39:17.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-03-24T02:39:17.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-03-24T02:39:17.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:17.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:17.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:17.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:17.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:17.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:17.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:17.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:17.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:17.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:17.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:17.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:17.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:17.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:17.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:17.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:17.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:17.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-03-24T02:39:17.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-03-24T02:39:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-03-24T02:39:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-03-24T02:39:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_12.status 2026-03-24T02:39:17.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:17.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:17.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-03-24T02:39:17.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:17.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:17.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:17.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:17.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:17.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:17.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:17.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=4, entry_tid=1' 2026-03-24T02:39:17.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-03-24T02:39:17.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:17.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:17.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:17.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-03-24T02:39:17.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:17.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:17.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:39:17.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:17.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-03-24T02:39:17.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_12.status 2026-03-24T02:39:17.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:17.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:17.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-03-24T02:39:17.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:17.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:17.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:17.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:17.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:17.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3511], [object_number=2, tag_tid=1, entry_tid=3510]]], state=connected] 2026-03-24T02:39:17.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.status 2026-03-24T02:39:17.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=1, tag_tid=4, entry_tid=1' 2026-03-24T02:39:17.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=4, entry_tid=1' -a 'object_number=1, tag_tid=4, entry_tid=1' = 'object_number=1, tag_tid=4, entry_tid=1' 2026-03-24T02:39:17.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:17.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_12 snap 2026-03-24T02:39:17.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:17.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:17.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:39:17.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:17.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:39:17.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:39:17.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:39:18.494 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.4051816642871726) > weight (0.25) 2026-03-24T02:39:18.494 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.4652678807970607) > weight (0.25) 2026-03-24T02:39:18.494 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-03-24T02:39:18.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-03-24T02:39:18.494 INFO:tasks.rbd_mirror_thrash:waiting for 0.32091576972438474 secs before reviving daemons 2026-03-24T02:39:18.495 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:39:18.493+0000 7f6fe0f85640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 54457) UID: 1000 2026-03-24T02:39:18.815 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-03-24T02:39:18.816 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:39:18.816 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-03-24T02:39:18.816 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-03-24T02:39:18.816 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-03-24T02:39:18.817 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-03-24T02:39:18.817 INFO:tasks.rbd_mirror_thrash:waiting for 96.77919489165056 secs before thrashing 2026-03-24T02:39:18.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_12@snap 2026-03-24T02:39:19.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_12': 2026-03-24T02:39:19.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:39:19.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:39:19.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:39:19.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 119e2db78904 2026-03-24T02:39:19.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.119e2db78904 2026-03-24T02:39:19.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:39:19.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:39:19.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:39:19.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:39:19.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:27:18 2026 2026-03-24T02:39:19.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:27:18 2026 2026-03-24T02:39:19.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:34:18 2026 2026-03-24T02:39:19.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:39:19.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 119e2db78904 2026-03-24T02:39:19.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:39:19.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:39:19.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 1ceb4471-c130-4ca5-8f54-c56dc41c52d6 2026-03-24T02:39:19.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:39:19.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:19.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_12 snap 2026-03-24T02:39:19.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:19.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:39:19.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:19.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:39:19.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.export 2026-03-24T02:39:19.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_12.export 2026-03-24T02:39:19.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_12.export 2026-03-24T02:39:19.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_12@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.export 2026-03-24T02:39:19.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:39:19.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_12@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_12.export 2026-03-24T02:39:19.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:39:19.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_12.export 2026-03-24T02:39:19.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_12.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_12.export 2026-03-24T02:39:19.508 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:39:19.501+0000 7fc159ee0640 -1 reset not still connected to 0x5641a57e6c30 2026-03-24T02:39:19.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:19.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:39:19.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_13 2026-03-24T02:39:19.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_13 snap 2026-03-24T02:39:19.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:19.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:19.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:39:19.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:19.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_13@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:39:19.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_13 2026-03-24T02:39:19.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:19.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:19.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:39:19.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_13 started 2026-03-24T02:39:19.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:19.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:19.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:39:19.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:39:19.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:39:19.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:19.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:39:19.909 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:19.901+0000 7f5496ce4640 -1 reset not still connected to 0x5652c7c3bad0 2026-03-24T02:39:20.509 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:39:20.501+0000 7fc159ee0640 -1 reset not still connected to 0x5641846f1a00 2026-03-24T02:39:20.510 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:39:20.501+0000 7fc159ee0640 -1 reset not still connected to 0x564185cc5a00 2026-03-24T02:39:20.515 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:39:20.509+0000 7fc159ee0640 -1 reset not still connected to 0x564199041ad0 2026-03-24T02:39:20.515 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:39:20.509+0000 7fc159ee0640 -1 reset not still connected to 0x56419d60f790 2026-03-24T02:39:20.526 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:39:20.521+0000 7fc159ee0640 -1 reset not still connected to 0x56419e1776c0 2026-03-24T02:39:20.526 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:39:20.521+0000 7fc159ee0640 -1 reset not still connected to 0x5641a41dea90 2026-03-24T02:39:20.526 INFO:tasks.ceph.cluster1.osd.0.vm06.stderr:2026-03-24T02:39:20.521+0000 7fc159ee0640 -1 reset not still connected to 0x5641a57e6c30 2026-03-24T02:39:20.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_13 started 2026-03-24T02:39:20.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:20.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:20.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:39:20.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:39:20.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:39:20.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:39:20.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_13 2026-03-24T02:39:20.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:39:20.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:39:20.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:20.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:20.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:39:20.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:39:20.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:20.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:39:20.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:20.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:39:20.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:20.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_13 2026-03-24T02:39:20.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:39:20.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:20.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:20.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:20.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:20.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:20.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:20.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:20.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:20.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:39:20.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:20.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:20.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:20.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:20.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_13 2026-03-24T02:39:20.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-03-24T02:39:20.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:39:20.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:20.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:20.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_13 2026-03-24T02:39:20.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:39:20.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:20.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:20.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:20.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:20.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:20.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:20.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:20.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:20.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:20.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:20.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:20.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:20.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:20.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:20.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:39:20.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:20.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:20.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:20.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:39:20.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:20.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:20.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:20.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:20.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:20.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:20.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:20.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:39:20.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:39:20.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:20.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:20.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:20.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:20.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_13 2026-03-24T02:39:20.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:39:20.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:39:20.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:39:20.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_13 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_13 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:39:20.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:20.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:39:21.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_13 2026-03-24T02:39:21.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:21.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:21.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:39:21.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:21.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_13 ']' 2026-03-24T02:39:21.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-03-24T02:39:21.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-03-24T02:39:21.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:21.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:21.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:21.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:21.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:21.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:21.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:21.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:21.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:21.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:21.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-03-24T02:39:21.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:21.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:21.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:21.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:21.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:21.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:21.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-03-24T02:39:21.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-03-24T02:39:21.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:21.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:21.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:21.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-03-24T02:39:21.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:21.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:21.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:21.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:21.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:21.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:21.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:21.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:21.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:21.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-03-24T02:39:21.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:21.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-03-24T02:39:21.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:21.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:21.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:39:21.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-03-24T02:39:21.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:21.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:21.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:39:21.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:21.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-03-24T02:39:21.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_13.status 2026-03-24T02:39:21.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:21.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:21.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-03-24T02:39:21.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:21.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:21.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:21.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:21.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:21.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:21.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:21.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:21.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-03-24T02:39:21.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:21.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:21.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:39:21.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-03-24T02:39:21.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:21.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:21.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:39:21.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:21.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-03-24T02:39:21.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_13.status 2026-03-24T02:39:21.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:21.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:21.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-03-24T02:39:21.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:21.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:21.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:21.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:21.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:21.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:21.286 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:21.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:21.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:21.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:39:21.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:39:21.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:39:21.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:21.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:21.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:39:21.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:21.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:39:21.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:21.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:39:21.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:21.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:39:21.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:21.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:39:21.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:21.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:39:21.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:21.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:39:21.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:21.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:39:21.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:39:21.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:39:21.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:21.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:39:21.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_13 2026-03-24T02:39:21.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:21.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:21.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:39:21.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:21.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_13 ']' 2026-03-24T02:39:21.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-03-24T02:39:21.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-03-24T02:39:21.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:21.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:21.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:21.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:21.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:21.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:21.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:21.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:21.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:21.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:21.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-03-24T02:39:21.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:21.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:21.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:21.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:21.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:21.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:21.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:21.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-03-24T02:39:21.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-03-24T02:39:21.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:21.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:21.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:21.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-03-24T02:39:21.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:21.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:21.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:21.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:21.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:21.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:21.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:21.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:21.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:21.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:21.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:21.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:21.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:21.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:21.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:21.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:21.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:21.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:21.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:21.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:21.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:21.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:21.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:21.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:21.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:21.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:21.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:21.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:21.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:21.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:21.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:21.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:21.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:21.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-03-24T02:39:21.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:21.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-03-24T02:39:21.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:21.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:21.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:39:21.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-03-24T02:39:21.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:21.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:21.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:39:21.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:21.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-03-24T02:39:21.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_13.status 2026-03-24T02:39:21.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:21.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:21.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-03-24T02:39:21.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:21.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:21.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:21.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:21.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:21.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:21.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:21.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:21.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-03-24T02:39:21.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:21.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:21.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:39:21.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-03-24T02:39:21.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:21.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:21.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:39:21.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:21.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-03-24T02:39:21.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_13.status 2026-03-24T02:39:21.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:21.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:21.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-03-24T02:39:21.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:21.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:21.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:21.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:21.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:21.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:21.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:39:22.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_13 2026-03-24T02:39:22.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:22.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:22.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:39:22.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:22.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_13 ']' 2026-03-24T02:39:22.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-03-24T02:39:22.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-03-24T02:39:22.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:22.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:22.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:22.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:22.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:22.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:22.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:22.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:22.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:22.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:22.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-03-24T02:39:22.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:22.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:22.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:22.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:22.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:22.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:22.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:22.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:22.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:22.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:22.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:22.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:22.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-03-24T02:39:22.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-03-24T02:39:22.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:22.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:22.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:22.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-03-24T02:39:22.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:22.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:22.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:22.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:22.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:22.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:22.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:22.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:22.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:22.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:22.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:22.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-03-24T02:39:22.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:22.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-03-24T02:39:22.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:22.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:22.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:39:22.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-03-24T02:39:22.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:22.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:22.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:39:22.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:22.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-03-24T02:39:22.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_13.status 2026-03-24T02:39:22.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:22.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:22.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-03-24T02:39:22.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:22.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:22.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:22.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:22.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:22.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:22.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:22.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:22.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-03-24T02:39:22.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:22.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:22.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:39:22.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-03-24T02:39:22.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:22.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:22.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:39:22.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:22.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-03-24T02:39:22.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_13.status 2026-03-24T02:39:22.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:22.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:22.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-03-24T02:39:22.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:22.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:22.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:22.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:22.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:22.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:22.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:22.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:22.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:22.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:22.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:22.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:39:23.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_13 2026-03-24T02:39:23.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:23.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:23.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:39:23.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:23.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_13 ']' 2026-03-24T02:39:23.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-03-24T02:39:23.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-03-24T02:39:23.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:23.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:23.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:23.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:23.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:23.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:23.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:23.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:23.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:23.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:23.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-03-24T02:39:23.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:23.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:23.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:23.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:23.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:23.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:23.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-03-24T02:39:23.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-03-24T02:39:23.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:23.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-03-24T02:39:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:23.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:23.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:23.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:23.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-03-24T02:39:23.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:23.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-03-24T02:39:23.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:23.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:23.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:39:23.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-03-24T02:39:23.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:23.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:23.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:39:23.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:23.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-03-24T02:39:23.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_13.status 2026-03-24T02:39:23.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-03-24T02:39:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:23.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:23.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:23.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:23.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-03-24T02:39:23.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:23.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:23.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:39:23.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-03-24T02:39:23.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:23.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:23.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:39:23.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:23.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-03-24T02:39:23.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_13.status 2026-03-24T02:39:23.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:23.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:23.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-03-24T02:39:23.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:23.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:23.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:23.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:23.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:23.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:23.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.status 2026-03-24T02:39:23.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:23.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:23.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:23.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_13 snap 2026-03-24T02:39:23.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:23.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:23.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:39:23.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:23.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:39:23.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:39:23.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:39:24.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_13@snap 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_13': 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c9be200542 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c9be200542 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:27:24 2026 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:27:24 2026 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:29:27 2026 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c9be200542 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: fed73411-e3f3-4bc6-af5b-ff6461688863 2026-03-24T02:39:24.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:39:24.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:24.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_13 snap 2026-03-24T02:39:24.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:24.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:39:24.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:24.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:39:24.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.export 2026-03-24T02:39:24.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_13.export 2026-03-24T02:39:24.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_13.export 2026-03-24T02:39:24.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_13@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.export 2026-03-24T02:39:24.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:39:24.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_13@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_13.export 2026-03-24T02:39:24.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:39:24.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_13.export 2026-03-24T02:39:24.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_13.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_13.export 2026-03-24T02:39:24.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:24.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:39:24.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_14 2026-03-24T02:39:24.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_14 snap 2026-03-24T02:39:24.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:24.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:24.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:39:24.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:24.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_14@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:39:25.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_14 2026-03-24T02:39:25.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:25.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:25.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:39:25.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_14 started 2026-03-24T02:39:25.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:25.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:25.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:39:25.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:39:25.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:39:25.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:25.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:39:26.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_14 started 2026-03-24T02:39:26.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:26.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:26.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:39:26.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:39:26.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:39:26.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:39:26.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_14 2026-03-24T02:39:26.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:39:26.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:39:26.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:26.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:26.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:39:26.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:39:26.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:26.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:39:26.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:26.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:39:26.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_14 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:26.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:26.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:39:26.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:26.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:26.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:26.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:26.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:26.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:26.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:26.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:39:26.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:39:26.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:26.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:26.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:26.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:26.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_14 2026-03-24T02:39:26.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-03-24T02:39:26.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:39:26.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:26.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:26.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_14 2026-03-24T02:39:26.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:39:26.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:26.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:26.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:26.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:26.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:26.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:26.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:26.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:26.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:26.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:26.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:26.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:26.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:26.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:39:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:39:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:39:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:39:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_14 2026-03-24T02:39:26.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-03-24T02:39:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:39:26.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:26.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_14 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:26.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:26.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:26.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:26.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:26.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:26.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:39:26.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:39:26.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:26.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:26.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:26.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:26.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_14 2026-03-24T02:39:26.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:39:26.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:39:26.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:39:26.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:39:26.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:39:26.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:39:26.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:39:26.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_14 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_14 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:26.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:39:27.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_14 2026-03-24T02:39:27.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:27.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:27.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:39:27.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:27.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_14 ']' 2026-03-24T02:39:27.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-03-24T02:39:27.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-03-24T02:39:27.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:27.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:27.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:27.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:27.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:27.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:27.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:27.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:27.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:27.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:27.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:27.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:27.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:27.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:27.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:27.149 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-03-24T02:39:27.173 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:39:27.165+0000 7fb2796c9640 -1 reset not still connected to 0x55ac14f7c000 2026-03-24T02:39:27.173 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:39:27.165+0000 7fb2796c9640 -1 reset not still connected to 0x55ac14f7c340 2026-03-24T02:39:27.173 INFO:tasks.ceph.cluster1.osd.1.vm06.stderr:2026-03-24T02:39:27.165+0000 7fb2796c9640 -1 reset not still connected to 0x55ac14f7c410 2026-03-24T02:39:27.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-03-24T02:39:27.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:27.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-03-24T02:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:27.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:27.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:27.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:27.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-03-24T02:39:27.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-03-24T02:39:27.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:27.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:27.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:27.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-03-24T02:39:27.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:27.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:27.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:27.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:27.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:27.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:27.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:27.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:27.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:27.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:27.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:27.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:27.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:27.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:27.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-03-24T02:39:27.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:27.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-03-24T02:39:27.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:27.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:27.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:27.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-03-24T02:39:27.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:27.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:27.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:27.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:27.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-03-24T02:39:27.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_14.status 2026-03-24T02:39:27.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:27.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:27.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-03-24T02:39:27.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:27.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:27.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:27.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:27.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:27.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:27.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:27.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:27.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-03-24T02:39:27.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:27.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:27.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:27.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-03-24T02:39:27.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:27.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:27.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:27.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:27.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-03-24T02:39:27.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_14.status 2026-03-24T02:39:27.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:27.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:27.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-03-24T02:39:27.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:27.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:27.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:27.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:27.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:27.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:27.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:27.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:27.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:27.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:39:27.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:39:27.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:39:27.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:27.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:27.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:39:27.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:27.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:39:27.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:27.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:39:27.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:27.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:39:27.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:27.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:39:27.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:27.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:39:27.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:27.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:39:27.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:27.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:39:27.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:39:27.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:39:27.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:27.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:39:27.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_14 2026-03-24T02:39:27.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:27.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:27.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:39:27.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:27.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_14 ']' 2026-03-24T02:39:27.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-03-24T02:39:27.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-03-24T02:39:27.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:27.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:27.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:27.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:27.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:27.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:27.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:27.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:27.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:27.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:27.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-03-24T02:39:27.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:27.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:27.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:27.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:27.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:27.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:27.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:27.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:27.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:27.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:27.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-03-24T02:39:27.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-03-24T02:39:27.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:27.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:27.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:27.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-03-24T02:39:27.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:27.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:27.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:27.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:27.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:27.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:27.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:27.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:27.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:27.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-03-24T02:39:27.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-03-24T02:39:27.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:27.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:27.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:27.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-03-24T02:39:27.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:27.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:27.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:27.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:27.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:27.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:27.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:27.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:27.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:27.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:27.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:27.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:27.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:27.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:27.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:27.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:27.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:27.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:27.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:27.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:27.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:27.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:27.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:27.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:27.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:27.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:27.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:27.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:27.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:27.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:27.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:27.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:27.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:27.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-03-24T02:39:27.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:27.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-03-24T02:39:27.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:27.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:27.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:27.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-03-24T02:39:27.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:27.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:27.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:27.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:27.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-03-24T02:39:27.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_14.status 2026-03-24T02:39:27.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:27.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:27.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-03-24T02:39:27.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:28.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:28.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:28.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:28.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:28.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:28.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:28.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:28.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-03-24T02:39:28.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:28.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:28.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:28.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-03-24T02:39:28.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:28.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:28.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:28.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:28.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-03-24T02:39:28.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_14.status 2026-03-24T02:39:28.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:28.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:28.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-03-24T02:39:28.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:28.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:28.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:28.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:28.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:28.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:28.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:28.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:28.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:28.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:28.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:28.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_14 2026-03-24T02:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_14 ']' 2026-03-24T02:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-03-24T02:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-03-24T02:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:28.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:28.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-03-24T02:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:28.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:28.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:28.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:28.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:28.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:28.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:28.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:28.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:28.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:28.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:28.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:28.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:28.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:28.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:28.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:28.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:28.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:28.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:28.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:28.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-03-24T02:39:28.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-03-24T02:39:28.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:28.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:28.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:28.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:28.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:28.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:28.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-03-24T02:39:28.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-03-24T02:39:28.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:28.616 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:28.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:28.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-03-24T02:39:28.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:28.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:28.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:28.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:28.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:28.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:28.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:28.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:28.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:28.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:28.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:28.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-03-24T02:39:28.708 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:28.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-03-24T02:39:28.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:28.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:28.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:28.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-03-24T02:39:28.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:28.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:28.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:28.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:28.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-03-24T02:39:28.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_14.status 2026-03-24T02:39:28.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:28.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:28.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-03-24T02:39:28.711 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:28.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:28.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:28.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:28.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:28.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:28.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:28.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:28.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-03-24T02:39:28.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:28.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:28.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:28.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-03-24T02:39:28.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:28.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:28.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:28.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:28.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-03-24T02:39:28.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_14.status 2026-03-24T02:39:28.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:28.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:28.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-03-24T02:39:28.753 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:28.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:39:29.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_14 2026-03-24T02:39:29.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:29.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:29.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:39:29.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:29.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_14 ']' 2026-03-24T02:39:29.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-03-24T02:39:29.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-03-24T02:39:29.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:29.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:29.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:29.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:29.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:29.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:29.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:29.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:29.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-03-24T02:39:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:29.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:29.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:29.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:29.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:29.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:29.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:29.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:29.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:29.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:29.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:29.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:29.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:29.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:29.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:29.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:29.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:29.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:29.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:29.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:29.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-03-24T02:39:29.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-03-24T02:39:29.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:29.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:29.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:29.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-03-24T02:39:29.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:29.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:29.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:29.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:29.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:29.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:29.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:29.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:29.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:29.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:29.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:29.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:29.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:29.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:29.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:29.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:29.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:29.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:29.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:29.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:29.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:29.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:29.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:29.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:29.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:29.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:29.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:29.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:29.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:29.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:29.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:29.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:29.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:29.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-03-24T02:39:29.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-03-24T02:39:29.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:29.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:29.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:29.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-03-24T02:39:29.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:29.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:29.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:29.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:29.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:29.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:29.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:29.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:29.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:29.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:29.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-03-24T02:39:29.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:29.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-03-24T02:39:29.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:29.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:29.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:29.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-03-24T02:39:29.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:29.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:29.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:29.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:29.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-03-24T02:39:29.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_14.status 2026-03-24T02:39:29.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:29.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:29.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-03-24T02:39:29.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:29.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:29.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:29.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:29.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:29.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:29.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:29.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:29.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-03-24T02:39:29.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:29.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:29.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:29.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-03-24T02:39:29.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:29.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:29.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:29.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:29.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-03-24T02:39:29.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_14.status 2026-03-24T02:39:29.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:29.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:29.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-03-24T02:39:29.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:29.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:29.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:29.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:29.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:29.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:29.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:29.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:29.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:29.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:29.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:29.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:39:31.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_14 2026-03-24T02:39:31.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:31.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:31.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:39:31.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:31.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_14 ']' 2026-03-24T02:39:31.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-03-24T02:39:31.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-03-24T02:39:31.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:31.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:31.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:31.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:31.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:31.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:31.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:31.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:31.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:31.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:31.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-03-24T02:39:31.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:31.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:31.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:31.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:31.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:31.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-03-24T02:39:31.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-03-24T02:39:31.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:31.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:31.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:31.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-03-24T02:39:31.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:31.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:31.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:31.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:31.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:31.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:31.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:31.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:31.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:31.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:31.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:31.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:31.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:31.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:31.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:31.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:31.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:31.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:31.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:31.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:31.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:31.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:31.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:31.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:31.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:31.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:31.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:31.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:31.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:31.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:31.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:31.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:31.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:31.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-03-24T02:39:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-03-24T02:39:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:31.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:31.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-03-24T02:39:31.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:31.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:31.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:31.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:31.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:31.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:31.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:31.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:31.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:31.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:31.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:31.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:31.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:31.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:31.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:31.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:31.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:31.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:31.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:31.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:31.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:31.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:31.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:31.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:31.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:31.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:31.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:31.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:31.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:31.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:31.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:31.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:31.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:31.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-03-24T02:39:31.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:31.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-03-24T02:39:31.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:31.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:31.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:31.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-03-24T02:39:31.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:31.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:31.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:31.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:31.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-03-24T02:39:31.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_14.status 2026-03-24T02:39:31.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:31.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:31.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-03-24T02:39:31.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:31.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:31.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:31.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-03-24T02:39:31.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:31.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:31.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:31.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-03-24T02:39:31.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:31.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:31.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:39:31.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:31.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-03-24T02:39:31.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_14.status 2026-03-24T02:39:31.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:31.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:31.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-03-24T02:39:31.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:31.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:31.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:31.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:31.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:31.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:31.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.status 2026-03-24T02:39:31.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:31.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:31.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:31.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_14 snap 2026-03-24T02:39:31.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:31.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:31.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:39:31.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:31.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:39:31.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:39:31.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:39:32.807 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:32.797+0000 7f5496ce4640 -1 reset not still connected to 0x5652c92ea5b0 2026-03-24T02:39:32.811 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:32.805+0000 7f5496ce4640 -1 reset not still connected to 0x5652d714f930 2026-03-24T02:39:32.821 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:32.813+0000 7f5496ce4640 -1 reset not still connected to 0x5652ebdffba0 2026-03-24T02:39:32.821 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:32.813+0000 7f5496ce4640 -1 reset not still connected to 0x5652ef236ea0 2026-03-24T02:39:32.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_14@snap 2026-03-24T02:39:32.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_14': 2026-03-24T02:39:32.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:39:32.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:39:32.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:39:32.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10d1fdcb45e7 2026-03-24T02:39:32.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10d1fdcb45e7 2026-03-24T02:39:32.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:39:32.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:39:32.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:39:32.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:39:32.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:27:30 2026 2026-03-24T02:39:32.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:27:30 2026 2026-03-24T02:39:32.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:29:27 2026 2026-03-24T02:39:32.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:39:32.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10d1fdcb45e7 2026-03-24T02:39:32.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:39:32.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:39:32.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 07c7a12a-23ff-4892-84d5-3c6818296cd2 2026-03-24T02:39:32.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:39:32.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:32.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_14 snap 2026-03-24T02:39:32.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:32.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:39:32.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:32.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:39:32.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.export 2026-03-24T02:39:32.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_14.export 2026-03-24T02:39:32.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_14.export 2026-03-24T02:39:32.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_14@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.export 2026-03-24T02:39:33.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:39:33.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_14@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_14.export 2026-03-24T02:39:33.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:39:33.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_14.export 2026-03-24T02:39:33.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_14.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_14.export 2026-03-24T02:39:33.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:33.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:39:33.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_15 2026-03-24T02:39:33.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_15 snap 2026-03-24T02:39:33.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:33.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:33.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:39:33.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:33.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_15@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:39:33.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_15 2026-03-24T02:39:33.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:33.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:33.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:39:33.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_15 started 2026-03-24T02:39:33.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:33.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:33.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:39:33.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:39:33.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:39:33.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:33.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:39:33.760 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:33.753+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c31ad0 2026-03-24T02:39:33.761 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:33.753+0000 7f5496ce4640 -1 reset not still connected to 0x5652d599b1e0 2026-03-24T02:39:33.761 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:33.753+0000 7f5496ce4640 -1 reset not still connected to 0x5652d5a3c820 2026-03-24T02:39:33.761 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:33.753+0000 7f5496ce4640 -1 reset not still connected to 0x5652d5c3f5f0 2026-03-24T02:39:33.761 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:33.753+0000 7f5496ce4640 -1 reset not still connected to 0x5652e4d71110 2026-03-24T02:39:33.761 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:39:33.753+0000 7f5496ce4640 -1 reset not still connected to 0x5652ef237c70 2026-03-24T02:39:34.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_15 started 2026-03-24T02:39:34.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:34.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:34.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:39:34.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:39:34.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:39:34.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:39:34.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_15 2026-03-24T02:39:34.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:39:34.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:34.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:34.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:39:34.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:39:34.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:34.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:39:34.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:39:34.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:34.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_15 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:39:34.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:39:34.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:34.733 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:34.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:34.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:34.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_15 2026-03-24T02:39:34.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-03-24T02:39:34.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:39:34.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_15 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:34.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:39:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:39:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:39:34.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:34.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:34.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:34.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:34.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_15 2026-03-24T02:39:34.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-03-24T02:39:34.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:39:34.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:34.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_15 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:34.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:39:34.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:39:34.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:34.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:34.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:34.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:34.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_15 2026-03-24T02:39:34.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:39:34.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:39:34.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:39:34.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:39:34.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_15 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_15 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:34.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:39:35.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_15 2026-03-24T02:39:35.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:35.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:35.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:39:35.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:35.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_15 ']' 2026-03-24T02:39:35.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_15' 2026-03-24T02:39:35.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2026-03-24T02:39:35.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:35.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:35.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:35.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:35.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:35.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:35.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:35.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:35.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:35.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:35.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2026-03-24T02:39:35.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:35.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:35.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:35.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:35.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:35.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:35.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:35.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:35.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:35.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:35.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:35.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:35.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:35.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:35.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:35.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:35.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:35.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:35.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:35.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:35.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:35.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:35.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:35.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:35.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:35.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:35.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:35.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:35.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:35.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:35.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:35.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:35.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:35.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2026-03-24T02:39:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-03-24T02:39:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:35.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:35.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:35.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:35.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:35.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:35.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:35.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:35.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:35.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:35.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:35.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:35.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:35.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:35.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:35.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:35.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:35.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:35.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2026-03-24T02:39:35.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-03-24T02:39:35.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:35.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:35.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:35.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_15 2026-03-24T02:39:35.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:35.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:35.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:35.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:35.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:35.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:35.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:35.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:35.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:35.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:35.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:35.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:35.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:35.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:35.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:35.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:35.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:35.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_15 2026-03-24T02:39:35.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:35.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_15 2026-03-24T02:39:35.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:35.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:35.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_15 2026-03-24T02:39:35.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_15 '' 2026-03-24T02:39:35.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:35.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:35.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_15 2026-03-24T02:39:35.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:35.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-03-24T02:39:35.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_15.status 2026-03-24T02:39:35.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:35.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_15.status 2026-03-24T02:39:35.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-03-24T02:39:35.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_15.status 2026-03-24T02:39:35.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:39:35.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:39:35.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:35.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:35.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:35.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_15.status 2026-03-24T02:39:35.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:39:35.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2026-03-24T02:39:35.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:35.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:35.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_15 2026-03-24T02:39:35.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2026-03-24T02:39:35.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:35.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:35.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_15 2026-03-24T02:39:35.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:35.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-03-24T02:39:35.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_15.status 2026-03-24T02:39:35.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:35.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_15.status 2026-03-24T02:39:35.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-03-24T02:39:35.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_15.status 2026-03-24T02:39:35.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:39:35.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:39:35.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:35.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:35.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:35.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_15.status 2026-03-24T02:39:35.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:39:35.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:39:35.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:39:35.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:39:35.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:39:35.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:39:35.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:39:35.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:39:35.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:35.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:39:35.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:39:35.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:39:35.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:35.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:39:35.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:39:35.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:39:35.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:35.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:39:35.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:39:35.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:39:35.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:35.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:39:35.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:39:35.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:39:35.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:35.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:39:35.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_15 2026-03-24T02:39:35.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:35.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:35.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:39:35.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:35.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_15 ']' 2026-03-24T02:39:35.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_15' 2026-03-24T02:39:35.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2026-03-24T02:39:35.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:35.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:35.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:35.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:35.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:35.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:35.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:35.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:35.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:35.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:35.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2026-03-24T02:39:35.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:35.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:35.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:35.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:35.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:35.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2026-03-24T02:39:35.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-03-24T02:39:35.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:35.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:35.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:35.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2026-03-24T02:39:35.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:35.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:35.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:35.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:35.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:35.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:35.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:35.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:35.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:35.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:35.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:35.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:35.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:35.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:35.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:35.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:35.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:35.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2026-03-24T02:39:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-03-24T02:39:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:35.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:35.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_15 2026-03-24T02:39:35.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:35.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:35.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:35.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:35.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:35.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:35.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:35.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:35.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:35.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:35.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:35.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:35.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:35.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:35.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:35.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:35.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:35.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:35.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:35.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:35.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:35.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:35.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:35.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:35.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:35.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:35.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:35.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:35.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:35.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:35.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:35.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:35.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:35.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_15 2026-03-24T02:39:35.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:35.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_15 2026-03-24T02:39:35.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:35.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:35.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_15 2026-03-24T02:39:35.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_15 '' 2026-03-24T02:39:35.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:35.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:35.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_15 2026-03-24T02:39:35.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:35.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-03-24T02:39:35.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_15.status 2026-03-24T02:39:35.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:35.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_15.status 2026-03-24T02:39:35.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-03-24T02:39:35.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_15.status 2026-03-24T02:39:36.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:39:36.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:39:36.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:36.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:36.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:36.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_15.status 2026-03-24T02:39:36.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:39:36.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2026-03-24T02:39:36.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:36.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:36.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_15 2026-03-24T02:39:36.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2026-03-24T02:39:36.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:36.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:36.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_15 2026-03-24T02:39:36.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:36.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-03-24T02:39:36.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_15.status 2026-03-24T02:39:36.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:36.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_15.status 2026-03-24T02:39:36.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-03-24T02:39:36.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_15.status 2026-03-24T02:39:36.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:39:36.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:39:36.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:36.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:36.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:36.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_15.status 2026-03-24T02:39:36.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:39:36.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:39:36.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:36.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_15 snap 2026-03-24T02:39:36.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:36.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:36.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:39:36.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:36.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:39:36.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:39:36.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:39:37.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_15@snap 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_15': 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 116dffed13bd 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.116dffed13bd 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:27:42 2026 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:27:42 2026 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:31:36 2026 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 116dffed13bd 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 366053e5-6417-4171-b189-a4ca10e42fae 2026-03-24T02:39:37.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:39:37.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:37.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_15 snap 2026-03-24T02:39:37.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:37.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:39:37.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:37.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:39:37.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_15.export 2026-03-24T02:39:37.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_15.export 2026-03-24T02:39:37.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_15.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_15.export 2026-03-24T02:39:37.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_15@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_15.export 2026-03-24T02:39:37.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:39:37.280 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_15@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_15.export 2026-03-24T02:39:37.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:39:37.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_15.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_15.export 2026-03-24T02:39:37.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_15.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_15.export 2026-03-24T02:39:37.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:37.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:39:37.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_16 2026-03-24T02:39:37.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_16 snap 2026-03-24T02:39:37.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:37.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:37.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:39:37.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:37.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_16@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:39:38.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_16 2026-03-24T02:39:38.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:38.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:38.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:39:38.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_16 started 2026-03-24T02:39:38.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:38.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:38.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:39:38.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:39:38.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:39:38.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:38.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:39:39.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_16 started 2026-03-24T02:39:39.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:39.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:39.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:39:39.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:39:39.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:39:39.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:39:39.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_16 2026-03-24T02:39:39.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:39:39.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:39:39.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:39.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:39.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:39:39.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:39:39.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:39.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:39:39.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:39.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:39.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:39.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:39.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:39.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:39:39.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:39:39.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:39.733 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:39.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:39.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:39.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2026-03-24T02:39:39.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-03-24T02:39:39.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:39:39.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_16 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:39:39.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:39:39.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:39.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:39.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:39.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:39.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_16 2026-03-24T02:39:39.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-03-24T02:39:39.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:39:39.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_16 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:39:39.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:39:39.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:39.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:39.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:39.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:39.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_16 2026-03-24T02:39:39.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-03-24T02:39:39.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:39:39.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_16 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:39:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:39:39.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:39:39.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:39.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:39.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:39.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_16 2026-03-24T02:39:40.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:39:40.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:39:40.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:39:40.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:39:40.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_16 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_16 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:40.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:39:40.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_16 2026-03-24T02:39:40.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:40.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:40.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:39:40.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:40.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_16 ']' 2026-03-24T02:39:40.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_16' 2026-03-24T02:39:40.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2026-03-24T02:39:40.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:40.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:40.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:40.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:40.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:40.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:40.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:40.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:40.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:40.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:40.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:40.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:40.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:40.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2026-03-24T02:39:40.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-03-24T02:39:40.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:40.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_16 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:40.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:40.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:40.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:40.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:40.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:40.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:40.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:40.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:40.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:40.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:40.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:40.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:40.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:40.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_16 2026-03-24T02:39:40.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-03-24T02:39:40.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:40.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_16 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:40.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:40.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:40.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:40.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:40.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_16 2026-03-24T02:39:40.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-03-24T02:39:40.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:40.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:40.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:40.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_16 2026-03-24T02:39:40.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:39:40.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:40.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:40.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:39:40.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:39:40.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:40.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:40.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:40.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:39:40.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:40.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:40.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:40.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:39:40.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:40.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:39:40.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:39:40.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:40.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:40.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:39:40.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:39:40.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:40.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:40.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:40.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:40.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:40.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:40.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:40.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:39:40.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:39:40.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:39:40.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:40.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:40.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:40.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_16 2026-03-24T02:39:40.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:40.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_16 2026-03-24T02:39:40.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:40.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:40.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_16 2026-03-24T02:39:40.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_16 '' 2026-03-24T02:39:40.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:40.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:40.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_16 2026-03-24T02:39:40.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:40.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-03-24T02:39:40.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_16.status 2026-03-24T02:39:40.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:40.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_16.status 2026-03-24T02:39:40.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-03-24T02:39:40.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_16.status 2026-03-24T02:39:40.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:40.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:40.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:40.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:40.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:40.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_16.status 2026-03-24T02:39:40.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:40.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2026-03-24T02:39:40.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:40.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:40.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_16 2026-03-24T02:39:40.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2026-03-24T02:39:40.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:40.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:40.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_16 2026-03-24T02:39:40.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:40.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-03-24T02:39:40.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_16.status 2026-03-24T02:39:40.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:40.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_16.status 2026-03-24T02:39:40.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-03-24T02:39:40.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_16.status 2026-03-24T02:39:40.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:40.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:40.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:40.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:40.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:40.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_16.status 2026-03-24T02:39:40.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:40.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:40.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:39:40.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:39:40.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:39:40.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:40.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:40.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:39:40.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:40.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:39:40.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:40.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:39:40.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:40.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:39:40.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:40.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:39:40.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:40.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:39:40.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:40.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:39:40.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:40.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:39:40.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:39:40.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:39:40.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:40.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:39:40.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_16 2026-03-24T02:39:40.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:40.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:40.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:39:40.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:40.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_16 ']' 2026-03-24T02:39:40.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_16' 2026-03-24T02:39:40.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2026-03-24T02:39:40.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:40.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:40.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:40.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:40.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:40.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:40.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:40.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:40.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:40.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:40.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:40.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:40.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:40.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:40.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2026-03-24T02:39:40.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-03-24T02:39:40.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:40.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_16 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:40.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:40.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:40.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:40.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:40.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:40.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:40.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:40.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:40.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_16 2026-03-24T02:39:40.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-03-24T02:39:40.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:40.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_16 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:40.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:40.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:40.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:40.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_16 2026-03-24T02:39:41.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-03-24T02:39:41.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:41.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:41.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:41.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_16 2026-03-24T02:39:41.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:39:41.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:41.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:41.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:39:41.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:39:41.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:41.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:41.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:41.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:39:41.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:41.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:41.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:41.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:39:41.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:41.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:39:41.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:39:41.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:41.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:41.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:39:41.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:39:41.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:41.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:41.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:41.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:41.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:41.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:41.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:41.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:39:41.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:39:41.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:39:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_16 2026-03-24T02:39:41.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:41.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_16 2026-03-24T02:39:41.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:41.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:41.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_16 2026-03-24T02:39:41.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_16 '' 2026-03-24T02:39:41.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:41.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:41.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_16 2026-03-24T02:39:41.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:41.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-03-24T02:39:41.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_16.status 2026-03-24T02:39:41.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:41.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_16.status 2026-03-24T02:39:41.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-03-24T02:39:41.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_16.status 2026-03-24T02:39:41.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:41.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:41.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:41.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:41.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:41.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_16.status 2026-03-24T02:39:41.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:41.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2026-03-24T02:39:41.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:41.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:41.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_16 2026-03-24T02:39:41.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2026-03-24T02:39:41.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:41.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:41.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_16 2026-03-24T02:39:41.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:41.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-03-24T02:39:41.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_16.status 2026-03-24T02:39:41.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:41.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_16.status 2026-03-24T02:39:41.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-03-24T02:39:41.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_16.status 2026-03-24T02:39:41.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:41.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:41.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:41.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:41.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:41.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_16.status 2026-03-24T02:39:41.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:41.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:41.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:41.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_16 snap 2026-03-24T02:39:41.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:41.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:41.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:39:41.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:41.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:39:41.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:39:41.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:39:42.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_16@snap 2026-03-24T02:39:42.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_16': 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 119e6f9cd439 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.119e6f9cd439 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:27:55 2026 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:27:55 2026 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:36:50 2026 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 119e6f9cd439 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: fdd8e2c9-d8d6-43c1-b493-e799b1c59a65 2026-03-24T02:39:42.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:39:42.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:42.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_16 snap 2026-03-24T02:39:42.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:42.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:39:42.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:42.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:39:42.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_16.export 2026-03-24T02:39:42.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_16.export 2026-03-24T02:39:42.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_16.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_16.export 2026-03-24T02:39:42.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_16@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_16.export 2026-03-24T02:39:42.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:39:42.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_16@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_16.export 2026-03-24T02:39:42.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:39:42.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_16.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_16.export 2026-03-24T02:39:42.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_16.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_16.export 2026-03-24T02:39:42.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:42.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:39:42.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_17 2026-03-24T02:39:42.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_17 snap 2026-03-24T02:39:42.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:42.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:42.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:39:42.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:42.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_17@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:39:42.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_17 2026-03-24T02:39:42.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:42.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:42.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:39:42.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_17 started 2026-03-24T02:39:42.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:42.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:42.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:39:42.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:39:42.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:39:42.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:42.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:39:43.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_17 started 2026-03-24T02:39:43.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:43.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:43.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:39:43.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:39:43.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:39:43.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:39:43.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_17 2026-03-24T02:39:43.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:39:43.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:39:43.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:43.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:43.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:39:43.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:39:43.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:43.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:39:43.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:43.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:39:43.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:43.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_17 2026-03-24T02:39:43.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:39:43.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:43.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:43.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:43.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:43.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:43.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:43.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:43.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:43.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:43.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:43.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:43.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:43.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:43.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:43.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:39:43.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:43.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:43.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:43.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:39:43.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:43.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:43.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:43.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:43.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:43.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:43.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:43.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:39:43.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:39:43.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:43.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:43.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:43.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:43.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_17 2026-03-24T02:39:44.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-03-24T02:39:44.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:39:44.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:44.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:44.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_17 2026-03-24T02:39:44.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:39:44.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:44.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:44.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:44.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:44.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:44.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:44.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:39:44.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:44.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:44.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:44.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:44.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_17 2026-03-24T02:39:44.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-03-24T02:39:44.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:39:44.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:44.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:44.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_17 2026-03-24T02:39:44.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:39:44.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:44.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:44.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:44.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:44.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:44.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:44.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:44.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:44.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:44.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:44.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:44.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:44.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:44.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:44.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:39:44.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:44.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:44.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:44.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:39:44.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:44.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:44.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:44.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:44.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:44.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:44.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:44.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:39:44.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:39:44.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:44.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:44.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:44.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:44.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_17 2026-03-24T02:39:44.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:39:44.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:39:44.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:39:44.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:39:44.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_17 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_17 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:44.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:39:44.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_17 2026-03-24T02:39:44.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:44.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:44.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:39:44.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:44.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_17 ']' 2026-03-24T02:39:44.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_17' 2026-03-24T02:39:44.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2026-03-24T02:39:44.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:44.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:44.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:44.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:44.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:44.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:44.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:44.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:44.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:44.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:44.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:44.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:44.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:44.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:44.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:44.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:44.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:44.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:44.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:44.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:44.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:44.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:44.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:44.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:44.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:44.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:44.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2026-03-24T02:39:44.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-03-24T02:39:44.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:44.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:44.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:44.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2026-03-24T02:39:44.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:44.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:44.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:44.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:44.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:44.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:44.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:44.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:44.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:44.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:44.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:44.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:44.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:44.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:44.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:44.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:44.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:44.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2026-03-24T02:39:44.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-03-24T02:39:44.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:44.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:44.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:44.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:44.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:44.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:44.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:44.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2026-03-24T02:39:44.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:44.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_17 2026-03-24T02:39:44.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:44.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:44.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_17 2026-03-24T02:39:44.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_17 '' 2026-03-24T02:39:44.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:44.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:44.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_17 2026-03-24T02:39:44.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:44.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-03-24T02:39:44.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_17.status 2026-03-24T02:39:44.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:44.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:44.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-03-24T02:39:44.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:44.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:44.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:44.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:44.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:44.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:44.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:44.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:44.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2026-03-24T02:39:44.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:44.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:44.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_17 2026-03-24T02:39:44.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2026-03-24T02:39:44.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:44.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:44.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_17 2026-03-24T02:39:44.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:44.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-03-24T02:39:44.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_17.status 2026-03-24T02:39:44.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:44.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:44.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-03-24T02:39:44.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:44.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:44.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:44.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:44.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:44.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:44.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:44.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:44.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:44.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:39:44.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:39:44.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:39:44.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:44.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:44.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:39:44.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:44.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:39:44.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:44.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:39:44.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:44.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:39:44.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:44.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:39:44.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:44.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:39:44.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:44.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:39:44.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:44.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:39:44.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:39:44.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:39:44.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:44.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:39:44.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_17 2026-03-24T02:39:44.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:44.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:44.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:39:44.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:44.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_17 ']' 2026-03-24T02:39:44.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_17' 2026-03-24T02:39:44.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2026-03-24T02:39:44.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:44.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:44.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:44.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:44.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:44.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:44.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:44.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:44.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:44.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:44.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:44.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:44.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:44.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2026-03-24T02:39:44.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-03-24T02:39:44.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:44.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:44.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:44.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:44.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:44.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:44.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:44.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2026-03-24T02:39:45.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-03-24T02:39:45.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:45.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:45.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:45.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2026-03-24T02:39:45.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:45.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:45.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:45.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:45.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:45.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:45.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:45.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:45.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:45.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:45.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:45.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:45.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:45.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:45.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:45.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:45.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2026-03-24T02:39:45.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:45.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_17 2026-03-24T02:39:45.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:45.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:45.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_17 2026-03-24T02:39:45.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_17 '' 2026-03-24T02:39:45.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:45.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:45.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_17 2026-03-24T02:39:45.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:45.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-03-24T02:39:45.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_17.status 2026-03-24T02:39:45.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:45.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:45.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-03-24T02:39:45.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:45.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:45.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:45.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:45.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:45.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:45.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:45.133 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:45.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2026-03-24T02:39:45.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:45.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:45.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_17 2026-03-24T02:39:45.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2026-03-24T02:39:45.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:45.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:45.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_17 2026-03-24T02:39:45.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:45.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-03-24T02:39:45.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_17.status 2026-03-24T02:39:45.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:45.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:45.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-03-24T02:39:45.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:45.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:45.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:45.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:45.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:45.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:45.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:45.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:45.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:45.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:45.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:45.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:39:45.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_17 2026-03-24T02:39:45.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:45.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:45.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:39:45.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:45.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_17 ']' 2026-03-24T02:39:45.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_17' 2026-03-24T02:39:45.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2026-03-24T02:39:45.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:45.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:45.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:45.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:45.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:45.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:45.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:45.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:45.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:45.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:45.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2026-03-24T02:39:45.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:45.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:45.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:45.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:45.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:45.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:45.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:45.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:45.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:45.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:45.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:45.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:45.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:45.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:45.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:45.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:45.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:45.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:45.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:45.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:45.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:45.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:45.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:45.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:45.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:45.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:45.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:45.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:45.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:45.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:45.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:45.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:45.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:45.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2026-03-24T02:39:45.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-03-24T02:39:45.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:45.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:45.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:45.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2026-03-24T02:39:45.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:45.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:45.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:45.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:45.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:45.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:45.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:45.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:45.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:45.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:45.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:45.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:45.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2026-03-24T02:39:45.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-03-24T02:39:45.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:45.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:45.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:45.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2026-03-24T02:39:45.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:45.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:45.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:45.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:45.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:45.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:45.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:45.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:45.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:45.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:45.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:45.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:45.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:45.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2026-03-24T02:39:45.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:45.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_17 2026-03-24T02:39:45.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:45.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:45.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_17 2026-03-24T02:39:45.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_17 '' 2026-03-24T02:39:45.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:45.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:45.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_17 2026-03-24T02:39:45.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:45.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-03-24T02:39:45.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_17.status 2026-03-24T02:39:45.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:45.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:45.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-03-24T02:39:45.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:45.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:45.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:45.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:45.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:45.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:45.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:45.833 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:45.833 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2026-03-24T02:39:45.833 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:45.833 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:45.833 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_17 2026-03-24T02:39:45.833 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2026-03-24T02:39:45.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:45.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:45.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_17 2026-03-24T02:39:45.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:45.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-03-24T02:39:45.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_17.status 2026-03-24T02:39:45.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:45.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:45.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-03-24T02:39:45.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:45.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:45.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:45.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:45.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:45.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:45.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.status 2026-03-24T02:39:45.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:45.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:45.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:45.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_17 snap 2026-03-24T02:39:45.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:45.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:45.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:39:45.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:45.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:39:45.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:39:45.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:39:46.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_17@snap 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_17': 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c92e380766 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c92e380766 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:28:02 2026 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:28:02 2026 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:31:22 2026 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c92e380766 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 3a12d77d-926c-4132-8317-1372d5435752 2026-03-24T02:39:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:39:46.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:46.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_17 snap 2026-03-24T02:39:46.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:46.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:39:46.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:46.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:39:46.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.export 2026-03-24T02:39:46.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_17.export 2026-03-24T02:39:46.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_17.export 2026-03-24T02:39:46.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_17@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.export 2026-03-24T02:39:47.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:39:47.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_17@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_17.export 2026-03-24T02:39:47.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:39:47.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_17.export 2026-03-24T02:39:47.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_17.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_17.export 2026-03-24T02:39:47.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:47.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:39:47.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_18 2026-03-24T02:39:47.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_18 snap 2026-03-24T02:39:47.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:47.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:47.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:39:47.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:47.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_18@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:39:47.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_18 2026-03-24T02:39:47.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:47.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:47.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:39:47.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_18 started 2026-03-24T02:39:47.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:47.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:47.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:39:47.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:39:47.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:39:47.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:47.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:39:48.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_18 started 2026-03-24T02:39:48.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:48.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:48.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:39:48.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:39:48.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:39:48.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:39:48.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_18 2026-03-24T02:39:48.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:39:48.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:48.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:48.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:39:48.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:39:48.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:39:48.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:48.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:39:48.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:48.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:39:48.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:48.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_18 2026-03-24T02:39:48.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:39:48.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:48.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:48.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:48.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:48.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:48.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:48.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:48.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:39:48.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:48.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:48.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:48.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:48.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_18 2026-03-24T02:39:49.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:49.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:39:49.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:49.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:49.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_18 2026-03-24T02:39:49.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:39:49.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:49.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:49.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:49.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:49.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:49.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:49.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:49.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:49.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:49.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:49.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:49.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:49.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:49.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:49.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:39:49.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:49.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:49.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:49.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:39:49.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:49.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:49.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:49.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:49.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:49.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:49.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:49.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:39:49.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:39:49.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:49.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:49.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:49.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:49.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_18 2026-03-24T02:39:49.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:49.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:39:49.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_18 2026-03-24T02:39:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:39:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:39:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:39:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:39:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:39:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:49.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:49.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:49.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:49.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_18 2026-03-24T02:39:49.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:39:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:49.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:49.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_18 2026-03-24T02:39:49.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:39:49.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:49.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:39:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:39:49.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:49.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:49.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:49.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_18 2026-03-24T02:39:49.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:39:49.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:39:49.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:39:49.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_18 2026-03-24T02:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:39:49.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_18 2026-03-24T02:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:49.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:39:49.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_18 2026-03-24T02:39:49.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:49.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:49.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:39:49.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:49.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_18 ']' 2026-03-24T02:39:49.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-03-24T02:39:49.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-03-24T02:39:49.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:49.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:49.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:49.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:49.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:49.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:49.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:49.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:49.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:49.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:49.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-03-24T02:39:49.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:49.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:49.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:49.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:49.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:49.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:49.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:49.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:49.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:49.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:49.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:49.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:49.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:49.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:49.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:49.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:49.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:49.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:49.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:49.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:49.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:49.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:49.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:49.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:49.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:49.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:49.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:49.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:49.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-03-24T02:39:49.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:49.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:49.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:49.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:49.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:49.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:49.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:49.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_18 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:49.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:49.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:49.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:49.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:49.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:49.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:39:49.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:39:49.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:39:49.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:49.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:49.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:49.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:49.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:49.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-03-24T02:39:49.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:49.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:49.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:49.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-03-24T02:39:49.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:49.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:49.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:49.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:49.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-03-24T02:39:49.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_18.status 2026-03-24T02:39:49.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:49.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:49.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-03-24T02:39:49.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:49.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:49.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:49.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:49.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:49.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:49.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:49.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:49.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-03-24T02:39:49.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:49.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:49.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:49.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-03-24T02:39:49.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:49.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:49.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:49.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:49.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-03-24T02:39:49.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_18.status 2026-03-24T02:39:49.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:49.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:49.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-03-24T02:39:49.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:49.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:49.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:49.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:49.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:49.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:49.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:49.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:39:49.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:39:49.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:39:49.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:39:49.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:39:49.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:39:49.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:49.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:39:49.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:49.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:39:49.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:39:49.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:39:49.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:49.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:39:49.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:49.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:39:49.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:49.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:39:49.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:39:49.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:39:49.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:49.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:39:49.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:39:49.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:39:49.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:49.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:39:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_18 2026-03-24T02:39:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:39:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_18 ']' 2026-03-24T02:39:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-03-24T02:39:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-03-24T02:39:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:50.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:50.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-03-24T02:39:50.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:50.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:50.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:50.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:50.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:50.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:50.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:50.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:50.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:50.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:50.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:50.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:50.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:50.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:50.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:50.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:50.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:50.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:50.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:50.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:50.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:50.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:50.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:50.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:50.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:50.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:50.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:50.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:50.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:50.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-03-24T02:39:50.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:50.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:50.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:50.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:50.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:50.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:50.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:50.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:50.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:50.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:50.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:50.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:50.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:50.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:50.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:50.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:50.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:50.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:50.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:50.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:50.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:50.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:50.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:50.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:50.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:50.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:50.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:50.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:50.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:50.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:50.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:50.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:50.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:50.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:50.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:50.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:50.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_18 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:39:50.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:39:50.280 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:50.280 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:50.280 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:50.280 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:50.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:50.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-03-24T02:39:50.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:50.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:50.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:50.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-03-24T02:39:50.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:50.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:50.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:50.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:50.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-03-24T02:39:50.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_18.status 2026-03-24T02:39:50.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:50.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:50.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-03-24T02:39:50.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:50.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:50.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:50.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:50.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:50.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:50.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:50.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:50.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-03-24T02:39:50.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:50.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:50.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:50.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-03-24T02:39:50.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:50.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:50.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:50.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:50.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-03-24T02:39:50.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_18.status 2026-03-24T02:39:50.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:50.390 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:50.390 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-03-24T02:39:50.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:50.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:50.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:50.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:50.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:50.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:50.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:50.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:39:50.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:39:50.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:39:50.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:50.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:39:50.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_18 2026-03-24T02:39:50.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:50.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:50.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:39:50.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:50.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_18 ']' 2026-03-24T02:39:50.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-03-24T02:39:50.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-03-24T02:39:50.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:50.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:50.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:50.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:50.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:50.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:50.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:50.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:50.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:50.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:50.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:50.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:50.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:50.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:50.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:50.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:50.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:50.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:50.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:50.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:50.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:50.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:50.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:50.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:50.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:50.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:50.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:50.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:50.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:50.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-03-24T02:39:50.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:50.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:50.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:50.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:50.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:50.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:50.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:50.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:50.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:50.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:50.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:50.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:50.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:50.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:50.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:50.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:50.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:50.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:50.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:50.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:50.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:50.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:50.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:50.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:50.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:50.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:50.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:50.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:50.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:50.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:50.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:50.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:50.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:50.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:50.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:50.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:50.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:50.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:50.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-03-24T02:39:50.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:50.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:50.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:50.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:50.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:50.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:50.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:50.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:51.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:51.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:51.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:51.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:51.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_18 2026-03-24T02:39:51.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:39:51.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:51.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:51.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:39:51.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:39:51.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:51.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:51.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:51.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:39:51.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:51.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:51.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:51.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:39:51.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:51.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:39:51.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:39:51.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:51.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:51.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:39:51.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:39:51.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:51.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:51.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:51.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:51.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:51.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:51.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:51.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:39:51.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:39:51.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:39:51.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:51.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:51.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:51.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:51.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:51.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-03-24T02:39:51.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:51.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:51.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:51.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-03-24T02:39:51.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:51.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:51.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:51.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:51.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-03-24T02:39:51.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_18.status 2026-03-24T02:39:51.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:51.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:51.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-03-24T02:39:51.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:51.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:51.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:51.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:51.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:51.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:51.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:51.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:51.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-03-24T02:39:51.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:51.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:51.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:51.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-03-24T02:39:51.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:51.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:51.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:51.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:51.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-03-24T02:39:51.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_18.status 2026-03-24T02:39:51.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:51.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:51.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-03-24T02:39:51.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:51.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:51.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:51.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:51.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:51.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:51.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:51.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:39:51.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:39:51.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:39:51.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:51.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:39:52.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_18 2026-03-24T02:39:52.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:52.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:52.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:39:52.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:52.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_18 ']' 2026-03-24T02:39:52.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-03-24T02:39:52.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-03-24T02:39:52.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:52.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:52.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:52.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:52.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:52.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:52.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:52.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:52.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:52.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:52.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:52.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:52.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:52.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:52.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:52.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:52.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:52.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:52.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:52.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:52.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:52.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:52.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:52.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:52.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:52.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:52.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:52.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:52.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:52.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:52.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:52.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-03-24T02:39:52.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:52.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:52.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:52.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:52.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:52.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:52.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:52.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:52.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:52.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:52.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:52.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:52.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:52.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:52.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:52.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:52.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:52.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:52.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:52.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:52.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:52.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:52.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:52.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:52.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:52.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:52.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:52.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:52.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:52.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:52.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:52.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:52.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:52.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:52.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:52.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:52.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-03-24T02:39:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:52.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:52.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:52.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:52.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:52.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:52.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:52.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:52.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:52.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:52.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:52.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:52.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:52.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:52.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:52.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:52.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:52.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:52.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:52.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:52.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:52.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:52.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:52.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_18 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:52.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:52.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:52.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:39:52.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:39:52.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:39:52.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:52.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:52.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:52.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:52.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:52.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-03-24T02:39:52.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:52.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:52.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:52.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-03-24T02:39:52.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:52.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:52.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:52.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:52.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-03-24T02:39:52.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_18.status 2026-03-24T02:39:52.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:52.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:52.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-03-24T02:39:52.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:52.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:52.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:52.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-03-24T02:39:52.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:52.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:52.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:52.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-03-24T02:39:52.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:52.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:52.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:52.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:52.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-03-24T02:39:52.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_18.status 2026-03-24T02:39:52.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:52.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:52.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-03-24T02:39:52.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:52.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:52.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:52.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:52.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:52.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:52.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:52.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:39:52.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:39:52.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:39:52.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:52.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:39:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_18 2026-03-24T02:39:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:39:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_18 ']' 2026-03-24T02:39:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-03-24T02:39:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-03-24T02:39:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:53.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:53.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:53.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:53.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-03-24T02:39:53.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:53.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:53.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:54.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:54.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:54.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:54.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:54.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:54.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:54.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-03-24T02:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:54.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:54.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:54.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:54.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:54.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:54.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:54.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:54.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:54.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:54.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-03-24T02:39:54.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:54.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:54.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:54.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:54.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:54.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:54.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:54.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:54.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:54.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:54.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:54.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-03-24T02:39:54.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:54.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:54.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:54.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_18 2026-03-24T02:39:54.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:39:54.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:54.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:54.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:39:54.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:39:54.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:54.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:54.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:54.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:39:54.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:54.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:54.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:54.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:39:54.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:39:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:39:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:39:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:39:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:39:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:39:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:39:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:39:54.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:39:54.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:54.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:54.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:39:54.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_18 2026-03-24T02:39:54.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:54.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-03-24T02:39:54.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:54.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:54.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:54.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-03-24T02:39:54.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:54.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:54.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:54.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:54.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-03-24T02:39:54.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_18.status 2026-03-24T02:39:54.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:54.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:54.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-03-24T02:39:54.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:54.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:54.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:54.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:54.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:54.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:54.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:54.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:54.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-03-24T02:39:54.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:54.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:54.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:54.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-03-24T02:39:54.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:54.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:54.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:39:54.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:54.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-03-24T02:39:54.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_18.status 2026-03-24T02:39:54.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:54.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:54.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-03-24T02:39:54.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:54.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:39:54.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:39:54.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:54.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:54.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:54.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.status 2026-03-24T02:39:54.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:54.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:39:54.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:54.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_18 snap 2026-03-24T02:39:54.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:54.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:54.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:39:54.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:54.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:39:54.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:39:54.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:39:55.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_18@snap 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_18': 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10d1b8fb29bf 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10d1b8fb29bf 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:28:14 2026 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:28:14 2026 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:32:26 2026 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10d1b8fb29bf 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: f53b820a-ab33-46b1-a825-a9835dd8e8ff 2026-03-24T02:39:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:39:55.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:55.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_18 snap 2026-03-24T02:39:55.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:55.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:39:55.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:55.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:39:55.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.export 2026-03-24T02:39:55.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_18.export 2026-03-24T02:39:55.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_18.export 2026-03-24T02:39:55.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_18@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.export 2026-03-24T02:39:55.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:39:55.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_18@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_18.export 2026-03-24T02:39:55.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:39:55.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_18.export 2026-03-24T02:39:55.746 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_18.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_18.export 2026-03-24T02:39:55.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:55.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:39:55.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_19 2026-03-24T02:39:55.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_19 snap 2026-03-24T02:39:55.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:55.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:55.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:39:55.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:39:55.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_19@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:39:55.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_19 2026-03-24T02:39:55.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:55.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:55.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:39:55.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_19 started 2026-03-24T02:39:55.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:55.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:55.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:39:55.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:39:55.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:39:55.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:55.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:39:56.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_19 started 2026-03-24T02:39:56.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:56.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:56.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:39:56.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:39:56.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:39:56.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:39:56.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_19 2026-03-24T02:39:56.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:39:56.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:56.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:56.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:39:56.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:39:56.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:56.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:39:56.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:39:56.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:56.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_19 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:56.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:56.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:56.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:39:56.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:39:56.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:56.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:56.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:56.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:56.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_19 2026-03-24T02:39:57.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-03-24T02:39:57.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:39:57.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_19 2026-03-24T02:39:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:39:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:39:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:39:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:39:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:39:57.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:57.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:57.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:57.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:57.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_19 2026-03-24T02:39:57.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-03-24T02:39:57.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:39:57.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:57.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:57.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_19 2026-03-24T02:39:57.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:39:57.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:39:57.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:57.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:57.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:57.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:39:57.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:57.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:57.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:57.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:57.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:39:57.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:57.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:39:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:39:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:39:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:39:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:39:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:39:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:39:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:39:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:39:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:39:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:39:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:57.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:57.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:57.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:57.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_19 2026-03-24T02:39:57.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:39:57.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:39:57.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:39:57.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:39:57.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_19 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_19 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:57.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:39:57.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_19 2026-03-24T02:39:57.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:57.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:57.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:39:57.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:57.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_19 ']' 2026-03-24T02:39:57.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-03-24T02:39:57.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-03-24T02:39:57.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:57.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:57.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:57.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:57.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:57.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:57.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:57.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:57.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:57.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:57.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-03-24T02:39:57.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:57.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:57.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:57.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:57.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:57.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:57.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:57.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:57.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:57.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:57.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:57.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:57.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:57.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:57.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:57.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:57.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:57.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:57.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:57.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:57.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:57.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:57.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:57.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:57.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:57.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:57.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:57.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:57.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:57.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:57.434 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:57.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:57.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:57.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-03-24T02:39:57.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-03-24T02:39:57.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:57.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-03-24T02:39:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:57.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:57.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:57.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:57.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-03-24T02:39:57.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-03-24T02:39:57.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:57.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:57.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:57.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:57.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:57.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:57.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:57.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:57.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-03-24T02:39:57.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:57.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-03-24T02:39:57.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:57.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:57.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:39:57.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-03-24T02:39:57.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:57.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:57.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:39:57.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:57.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-03-24T02:39:57.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_19.status 2026-03-24T02:39:57.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:57.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:57.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-03-24T02:39:57.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:57.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:57.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:57.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:57.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:57.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:57.700 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:57.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:57.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-03-24T02:39:57.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:57.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:57.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:39:57.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-03-24T02:39:57.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:57.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:57.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:39:57.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:57.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-03-24T02:39:57.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_19.status 2026-03-24T02:39:57.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:57.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:57.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-03-24T02:39:57.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:57.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:57.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:57.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:57.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:57.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:57.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:57.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:57.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:39:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:39:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:39:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:39:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:57.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:39:57.745 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:57.745 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:39:57.745 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:57.746 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:39:57.746 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:57.746 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:39:57.747 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:57.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:39:57.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:57.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:39:57.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:39:57.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:39:57.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:39:57.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:39:57.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:57.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_19 2026-03-24T02:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_19 ']' 2026-03-24T02:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-03-24T02:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-03-24T02:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-03-24T02:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:57.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:57.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:57.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:57.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:57.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-03-24T02:39:58.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-03-24T02:39:58.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:58.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:58.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:58.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:58.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:58.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:58.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:58.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:58.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-03-24T02:39:58.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-03-24T02:39:58.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:58.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:58.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:58.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-03-24T02:39:58.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:58.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:58.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:58.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:58.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:58.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:58.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:58.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:58.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:58.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:58.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:58.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:58.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:58.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:58.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:58.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:58.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:58.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:58.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:58.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-03-24T02:39:58.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:58.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-03-24T02:39:58.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:58.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:58.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:39:58.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-03-24T02:39:58.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:58.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:58.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:39:58.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:58.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-03-24T02:39:58.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_19.status 2026-03-24T02:39:58.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:58.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:58.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-03-24T02:39:58.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:58.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:58.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:58.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:58.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:58.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:58.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:58.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:58.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-03-24T02:39:58.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:58.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:58.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:39:58.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-03-24T02:39:58.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:58.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:58.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:39:58.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:58.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-03-24T02:39:58.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_19.status 2026-03-24T02:39:58.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:58.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:58.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-03-24T02:39:58.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:58.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:58.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:58.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:58.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:58.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:58.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:58.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:39:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_19 2026-03-24T02:39:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:39:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_19 ']' 2026-03-24T02:39:58.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-03-24T02:39:58.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-03-24T02:39:58.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:58.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:58.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:58.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:58.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:58.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:58.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:58.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:58.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:58.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:58.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-03-24T02:39:58.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:58.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:58.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:58.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:58.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:58.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:58.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:58.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-03-24T02:39:58.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-03-24T02:39:58.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:58.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:58.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:58.745 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:58.745 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:58.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:58.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:58.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:58.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-03-24T02:39:58.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-03-24T02:39:58.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:58.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:58.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:58.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-03-24T02:39:58.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:58.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:58.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:58.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:58.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:58.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:58.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:58.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:58.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:58.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:58.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:58.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:58.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-03-24T02:39:58.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:39:58.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-03-24T02:39:58.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:58.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:58.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:39:58.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-03-24T02:39:58.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:58.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:58.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:39:58.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:39:58.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-03-24T02:39:58.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:58.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_19.status 2026-03-24T02:39:58.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:58.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-03-24T02:39:58.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:58.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:58.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:58.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:58.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:58.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:58.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:58.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:39:58.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-03-24T02:39:58.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:58.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:58.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:39:58.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-03-24T02:39:58.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:39:58.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:39:58.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:39:58.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:39:58.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-03-24T02:39:58.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_19.status 2026-03-24T02:39:58.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:39:58.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:58.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-03-24T02:39:58.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:58.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:39:58.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:39:58.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:39:58.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:39:58.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:39:58.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:39:58.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:58.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:58.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:39:58.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:39:58.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:39:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_19 2026-03-24T02:39:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:39:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:39:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:39:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_19 ']' 2026-03-24T02:39:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-03-24T02:39:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-03-24T02:39:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:39:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:39:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:39:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:39:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:39:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:39:59.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:39:59.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:39:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-03-24T02:39:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:39:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:39:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:39:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:39:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:39:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:39:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:39:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:39:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:39:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:39:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:39:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:39:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:39:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:39:59.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:59.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:59.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:39:59.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-03-24T02:39:59.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-03-24T02:39:59.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:59.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:59.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:59.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-03-24T02:39:59.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:39:59.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:59.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:59.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:39:59.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:39:59.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:59.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:59.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:59.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:39:59.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:59.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:59.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:59.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:39:59.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:59.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:39:59.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:39:59.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:59.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:59.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:39:59.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:39:59.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:59.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:59.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:59.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:39:59.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:39:59.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:59.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:59.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:39:59.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:39:59.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:39:59.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:59.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:59.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:39:59.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-03-24T02:39:59.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-03-24T02:39:59.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:39:59.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:39:59.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:39:59.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-03-24T02:39:59.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:39:59.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:39:59.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:39:59.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:39:59.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:39:59.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:39:59.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:39:59.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:39:59.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:39:59.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:59.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:39:59.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:59.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:39:59.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:39:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:39:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:39:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:39:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:39:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:39:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:39:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:39:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:39:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:39:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:39:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:39:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:39:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:39:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:39:59.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:39:59.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:59.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:59.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:39:59.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-03-24T02:40:00.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:00.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-03-24T02:40:00.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:00.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:00.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:40:00.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-03-24T02:40:00.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:00.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:00.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:40:00.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:00.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-03-24T02:40:00.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_19.status 2026-03-24T02:40:00.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:00.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:40:00.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-03-24T02:40:00.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:40:00.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:00.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:00.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:00.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:00.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:40:00.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-03-24T02:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-03-24T02:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-03-24T02:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_19.status 2026-03-24T02:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:00.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:40:00.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-03-24T02:40:00.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:40:00.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:00.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:00.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:00.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:00.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:00.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:40:00.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:00.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:00.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:00.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:00.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:40:00.241 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8867795300207435) > weight (0.25) 2026-03-24T02:40:00.241 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.8787954227322727) > weight (0.25) 2026-03-24T02:40:00.241 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.5712575941695309) > weight (0.25) 2026-03-24T02:40:00.241 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.6441172776852514) > weight (0.25) 2026-03-24T02:40:00.241 INFO:tasks.rbd_mirror_thrash:waiting for 62.898456248392485 secs before thrashing 2026-03-24T02:40:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_19 2026-03-24T02:40:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:40:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_19 ']' 2026-03-24T02:40:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-03-24T02:40:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-03-24T02:40:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:01.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:01.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:01.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:01.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-03-24T02:40:01.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:01.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:01.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:01.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:01.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:01.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:01.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:01.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:01.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:01.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:01.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:01.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:01.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:01.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:01.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:01.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:01.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:01.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:01.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:01.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:01.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:01.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:01.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:01.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:01.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:01.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:01.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:01.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:01.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:01.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:01.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:01.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:01.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:01.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-03-24T02:40:01.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-03-24T02:40:01.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:01.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:01.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:01.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:01.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:01.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:01.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-03-24T02:40:01.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-03-24T02:40:01.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:01.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:01.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:01.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:01.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:01.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:01.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:01.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:01.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:01.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:01.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-03-24T02:40:01.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:01.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-03-24T02:40:01.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:01.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:01.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:40:01.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-03-24T02:40:01.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:01.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:01.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:40:01.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:01.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-03-24T02:40:01.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_19.status 2026-03-24T02:40:01.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:01.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:40:01.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-03-24T02:40:01.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:40:01.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:01.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:01.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:01.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:01.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:01.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:40:01.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:01.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-03-24T02:40:01.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:01.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:01.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:40:01.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-03-24T02:40:01.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:01.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:01.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:40:01.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:01.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-03-24T02:40:01.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_19.status 2026-03-24T02:40:01.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:01.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:40:01.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-03-24T02:40:01.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:40:01.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:01.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:01.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:01.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:01.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:02.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.status 2026-03-24T02:40:02.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:02.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:02.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:02.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_19 snap 2026-03-24T02:40:02.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:02.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:02.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:40:02.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:02.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:40:02.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:40:02.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:40:03.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_19@snap 2026-03-24T02:40:03.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_19': 2026-03-24T02:40:03.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:40:03.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:40:03.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:40:03.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 116d31689b69 2026-03-24T02:40:03.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.116d31689b69 2026-03-24T02:40:03.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:40:03.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:40:03.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:40:03.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:40:03.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:28:18 2026 2026-03-24T02:40:03.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:28:18 2026 2026-03-24T02:40:03.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:32:42 2026 2026-03-24T02:40:03.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:40:03.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 116d31689b69 2026-03-24T02:40:03.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:40:03.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:40:03.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 8e6d96f0-1b12-42c7-a33b-1aa2094e045f 2026-03-24T02:40:03.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:40:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_19 snap 2026-03-24T02:40:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:40:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:40:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.export 2026-03-24T02:40:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_19.export 2026-03-24T02:40:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_19.export 2026-03-24T02:40:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_19@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.export 2026-03-24T02:40:03.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:40:03.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_19@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_19.export 2026-03-24T02:40:03.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:40:03.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_19.export 2026-03-24T02:40:03.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_19.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_19.export 2026-03-24T02:40:03.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:03.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:40:03.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_20 2026-03-24T02:40:03.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_20 snap 2026-03-24T02:40:03.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:03.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:03.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:40:03.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:03.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_20@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:40:04.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_20 2026-03-24T02:40:04.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:04.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:04.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:40:04.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_20 started 2026-03-24T02:40:04.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:04.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:04.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:40:04.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:40:04.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:40:04.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:04.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:40:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_20 started 2026-03-24T02:40:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:40:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:40:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:40:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:40:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_20 2026-03-24T02:40:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:40:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:40:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:40:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:40:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:40:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_20 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:05.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:05.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:05.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:05.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_20 2026-03-24T02:40:05.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-03-24T02:40:05.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:40:05.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_20 2026-03-24T02:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:05.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:40:05.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:05.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:05.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:05.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:40:05.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:05.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:05.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:05.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:05.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:05.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:05.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:05.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:40:05.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:40:05.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:05.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:05.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:05.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:05.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_20 2026-03-24T02:40:05.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-03-24T02:40:05.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:40:05.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:05.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:05.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_20 2026-03-24T02:40:05.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:40:05.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:05.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:05.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:05.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:05.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:05.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:05.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:05.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:05.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:05.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:05.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:05.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:05.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:40:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:40:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:40:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:40:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:05.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:05.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:05.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:05.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_20 2026-03-24T02:40:05.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-03-24T02:40:05.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:40:05.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_20 2026-03-24T02:40:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:40:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:40:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:40:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:40:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:40:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:40:05.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:40:05.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:40:05.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:05.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:05.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:40:05.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:40:05.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:05.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:05.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:05.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:40:05.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:05.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:05.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:05.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:40:05.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:40:05.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:40:05.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:05.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:05.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:05.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_20 2026-03-24T02:40:05.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:40:05.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:40:05.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:40:05.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:40:05.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:40:05.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:40:05.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:40:05.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:05.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_20 2026-03-24T02:40:05.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:40:05.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:05.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:40:05.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:40:05.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:40:05.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:40:05.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_20 2026-03-24T02:40:05.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:40:05.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:05.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:40:05.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:40:05.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:40:05.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:40:05.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:40:05.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:40:05.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:05.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:40:05.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_20 2026-03-24T02:40:05.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:05.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:05.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:40:05.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:05.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_20 ']' 2026-03-24T02:40:05.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_20' 2026-03-24T02:40:05.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2026-03-24T02:40:05.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:05.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:05.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:05.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:05.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:05.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:05.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:05.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:05.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:05.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:05.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2026-03-24T02:40:05.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:05.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:05.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:05.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:05.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:05.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:05.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:05.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:05.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:05.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:05.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:05.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:05.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:05.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:05.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:05.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:05.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:05.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:05.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:05.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2026-03-24T02:40:05.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-03-24T02:40:05.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:05.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:05.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:05.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_20 2026-03-24T02:40:05.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:05.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:05.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:05.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:05.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:05.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:05.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:05.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:05.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:05.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:05.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:05.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:05.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:05.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:05.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:05.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:05.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:05.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:05.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:05.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:05.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:05.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:05.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:05.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:05.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:05.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:05.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:05.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:05.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:05.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:05.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:05.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:05.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:05.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_20 2026-03-24T02:40:05.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-03-24T02:40:05.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:05.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:05.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:05.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_20 2026-03-24T02:40:05.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:05.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:05.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:05.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:05.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:05.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:05.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:05.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:05.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:05.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:05.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:05.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:05.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_20 2026-03-24T02:40:05.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-03-24T02:40:05.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:05.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_20 2026-03-24T02:40:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:40:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:40:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:40:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:40:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:40:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:40:05.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:40:05.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:40:05.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:05.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:05.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:40:05.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:40:05.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:05.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:05.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:05.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:40:05.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:05.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:05.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:05.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:40:05.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:40:05.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:40:05.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:05.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:05.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:05.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_20 2026-03-24T02:40:05.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:05.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_20 2026-03-24T02:40:05.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:05.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:05.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_20 2026-03-24T02:40:05.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_20 '' 2026-03-24T02:40:05.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:05.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:05.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_20 2026-03-24T02:40:05.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:05.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-03-24T02:40:05.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_20.status 2026-03-24T02:40:05.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:05.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_20.status 2026-03-24T02:40:05.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-03-24T02:40:05.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_20.status 2026-03-24T02:40:06.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:06.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:06.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:06.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:06.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:06.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_20.status 2026-03-24T02:40:06.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:06.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2026-03-24T02:40:06.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:06.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:06.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_20 2026-03-24T02:40:06.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2026-03-24T02:40:06.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:06.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:06.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_20 2026-03-24T02:40:06.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:06.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-03-24T02:40:06.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_20.status 2026-03-24T02:40:06.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:06.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_20.status 2026-03-24T02:40:06.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-03-24T02:40:06.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_20.status 2026-03-24T02:40:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:06.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_20.status 2026-03-24T02:40:06.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:06.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:06.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:40:06.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:40:06.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:40:06.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:06.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:06.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:40:06.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:06.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:40:06.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:06.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:40:06.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:06.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:40:06.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:06.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:40:06.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:06.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:40:06.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:06.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:40:06.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:06.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:40:06.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:40:06.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:40:06.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:06.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:40:06.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_20 2026-03-24T02:40:06.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:06.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:06.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:40:06.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:06.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_20 ']' 2026-03-24T02:40:06.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_20' 2026-03-24T02:40:06.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2026-03-24T02:40:06.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:06.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:06.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:06.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:06.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:06.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:06.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:06.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:06.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:06.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:06.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:06.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:06.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:06.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:06.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:06.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:06.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:06.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:06.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:06.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:06.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:06.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:06.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:06.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:06.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:06.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:06.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2026-03-24T02:40:06.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-03-24T02:40:06.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:06.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_20 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:06.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:06.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:06.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:06.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:06.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:06.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:06.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:06.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:06.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:06.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_20 2026-03-24T02:40:06.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-03-24T02:40:06.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:06.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_20 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:06.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:06.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:06.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:06.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:06.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:06.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_20 2026-03-24T02:40:06.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-03-24T02:40:06.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:06.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:06.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:06.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_20 2026-03-24T02:40:06.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:40:06.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:06.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:06.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:40:06.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:40:06.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:06.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:06.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:06.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:40:06.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:40:06.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:06.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:06.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:06.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_20 2026-03-24T02:40:06.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:06.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_20 2026-03-24T02:40:06.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:06.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:06.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_20 2026-03-24T02:40:06.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_20 '' 2026-03-24T02:40:06.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:06.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:06.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_20 2026-03-24T02:40:06.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:06.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-03-24T02:40:06.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_20.status 2026-03-24T02:40:06.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:06.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_20.status 2026-03-24T02:40:06.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-03-24T02:40:06.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_20.status 2026-03-24T02:40:06.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:06.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:06.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:06.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:06.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:06.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_20.status 2026-03-24T02:40:06.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:06.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2026-03-24T02:40:06.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:06.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:06.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_20 2026-03-24T02:40:06.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2026-03-24T02:40:06.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:06.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:06.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_20 2026-03-24T02:40:06.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:06.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-03-24T02:40:06.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_20.status 2026-03-24T02:40:06.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:06.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_20.status 2026-03-24T02:40:06.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-03-24T02:40:06.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_20.status 2026-03-24T02:40:06.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:06.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:06.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:06.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:06.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:06.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_20.status 2026-03-24T02:40:06.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:06.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:06.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:06.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_20 snap 2026-03-24T02:40:06.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:06.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:06.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:40:06.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:06.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:40:06.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:40:06.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:40:07.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_20@snap 2026-03-24T02:40:07.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_20': 2026-03-24T02:40:07.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:40:07.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:40:07.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:40:07.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c953f42c03 2026-03-24T02:40:07.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c953f42c03 2026-03-24T02:40:07.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:40:07.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:40:07.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:40:07.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:40:07.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:30:54 2026 2026-03-24T02:40:07.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:30:54 2026 2026-03-24T02:40:07.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:37:11 2026 2026-03-24T02:40:07.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:40:07.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c953f42c03 2026-03-24T02:40:07.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:40:07.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:40:07.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: fb0f0a86-fc1a-4ffd-844c-8b7ff45aefbc 2026-03-24T02:40:07.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:40:07.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:07.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_20 snap 2026-03-24T02:40:07.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:07.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:40:07.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:07.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:40:07.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_20.export 2026-03-24T02:40:07.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_20.export 2026-03-24T02:40:07.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_20.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_20.export 2026-03-24T02:40:07.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_20@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_20.export 2026-03-24T02:40:07.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:40:07.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_20@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_20.export 2026-03-24T02:40:08.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:40:08.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_20.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_20.export 2026-03-24T02:40:08.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_20.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_20.export 2026-03-24T02:40:08.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:08.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:40:08.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_21 2026-03-24T02:40:08.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_21 snap 2026-03-24T02:40:08.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:08.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:08.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:40:08.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:08.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_21@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:40:09.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_21 2026-03-24T02:40:09.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:09.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:09.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:40:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_21 started 2026-03-24T02:40:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:40:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:40:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:40:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:40:10.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_21 started 2026-03-24T02:40:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:40:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:40:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:40:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:40:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_21 2026-03-24T02:40:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:40:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:40:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:40:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:40:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:40:10.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:10.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:40:10.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:10.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_21 2026-03-24T02:40:10.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:40:10.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:10.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:10.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:10.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:10.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:10.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:10.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:10.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:10.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:40:10.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:40:10.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:10.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:10.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:10.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:10.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_21 2026-03-24T02:40:10.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-03-24T02:40:10.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:40:10.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:10.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:10.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_21 2026-03-24T02:40:10.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:40:10.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:10.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:10.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:10.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:10.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:10.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:10.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:10.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:40:10.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:40:10.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:10.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:10.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:10.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:10.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_21 2026-03-24T02:40:10.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:40:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:40:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:40:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:40:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:40:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:40:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:40:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_21 2026-03-24T02:40:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:40:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:40:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:40:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:40:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:40:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_21 2026-03-24T02:40:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:40:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:40:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:40:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:40:10.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:40:10.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:40:10.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:40:10.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:10.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:40:10.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_21 2026-03-24T02:40:10.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:10.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:10.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:40:10.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:10.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_21 ']' 2026-03-24T02:40:10.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-03-24T02:40:10.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-03-24T02:40:10.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:10.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:10.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:10.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:10.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:10.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:10.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:10.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:10.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:10.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:10.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:10.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:10.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:10.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:10.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:10.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-03-24T02:40:10.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-03-24T02:40:10.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:10.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:10.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:10.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_21 2026-03-24T02:40:10.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:10.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:10.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:10.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:10.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:10.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:10.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:10.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:10.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:10.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:10.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:10.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:10.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:10.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:10.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:10.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:10.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:10.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:10.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:10.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:10.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:10.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:10.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:10.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:10.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:10.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:10.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:10.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:10.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:10.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:10.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:10.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:10.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:10.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_21 2026-03-24T02:40:10.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:10.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-03-24T02:40:10.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:10.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:10.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:10.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-03-24T02:40:10.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:10.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:10.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:10.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:10.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-03-24T02:40:10.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_21.status 2026-03-24T02:40:10.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:10.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:10.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-03-24T02:40:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:10.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:10.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:10.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-03-24T02:40:10.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:10.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:10.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:10.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-03-24T02:40:10.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:10.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:10.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:10.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-03-24T02:40:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_21.status 2026-03-24T02:40:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:10.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:10.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-03-24T02:40:10.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:10.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:10.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:10.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:10.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:10.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:10.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:10.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:10.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:10.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:40:10.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:40:10.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:40:10.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:10.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:10.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:40:10.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:10.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:40:10.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:10.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:40:10.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:10.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:40:10.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:10.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:40:10.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:10.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:40:10.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:10.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:40:10.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:10.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:40:10.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:40:10.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:40:10.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:10.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:40:11.187 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_21 2026-03-24T02:40:11.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:11.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:11.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:40:11.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:11.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_21 ']' 2026-03-24T02:40:11.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-03-24T02:40:11.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-03-24T02:40:11.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:11.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:11.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:11.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:11.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:11.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:11.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:11.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:11.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:11.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:11.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-03-24T02:40:11.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:11.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:11.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:11.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:11.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:11.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:11.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:11.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:11.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:11.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:11.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:11.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:11.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:11.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:11.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:11.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:11.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:11.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:11.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:11.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:11.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:11.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:11.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:11.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:11.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:11.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:11.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:11.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:11.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:11.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:11.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:11.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:11.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:11.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-03-24T02:40:11.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-03-24T02:40:11.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:11.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:11.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:11.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_21 2026-03-24T02:40:11.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:11.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:11.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:11.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:11.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:11.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:11.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:11.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:11.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:11.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:11.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:11.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:11.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:11.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:11.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:11.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:11.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:11.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:11.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_21 2026-03-24T02:40:11.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:11.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-03-24T02:40:11.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:11.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:11.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:11.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-03-24T02:40:11.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:11.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:11.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:11.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:11.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-03-24T02:40:11.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_21.status 2026-03-24T02:40:11.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:11.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:11.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-03-24T02:40:11.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:11.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:11.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:11.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:11.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:11.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:11.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:11.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:11.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-03-24T02:40:11.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:11.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:11.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:11.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-03-24T02:40:11.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:11.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:11.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:11.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:11.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-03-24T02:40:11.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_21.status 2026-03-24T02:40:11.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:11.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:11.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-03-24T02:40:11.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:11.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:11.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:11.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:11.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:11.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:11.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:11.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:11.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:11.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:11.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:11.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:40:11.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_21 2026-03-24T02:40:11.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:11.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:11.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:40:11.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:11.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_21 ']' 2026-03-24T02:40:11.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-03-24T02:40:11.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-03-24T02:40:11.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:11.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:11.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:11.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:11.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:11.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:11.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:11.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:11.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:11.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:11.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:11.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:11.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:11.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:11.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:11.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:11.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:11.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:11.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:11.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:11.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:11.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:11.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:11.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:11.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:11.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:11.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:11.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-03-24T02:40:11.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-03-24T02:40:11.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:11.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_21 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:11.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:11.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:11.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:11.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:11.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_21 2026-03-24T02:40:11.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:11.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-03-24T02:40:11.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:11.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:11.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:11.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-03-24T02:40:11.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:11.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:11.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:11.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:11.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-03-24T02:40:11.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_21.status 2026-03-24T02:40:11.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:11.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:11.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-03-24T02:40:11.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:11.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:11.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:11.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:11.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:11.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:11.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:12.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:12.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-03-24T02:40:12.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:12.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:12.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:12.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-03-24T02:40:12.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:12.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:12.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:12.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:12.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-03-24T02:40:12.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_21.status 2026-03-24T02:40:12.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:12.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:12.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-03-24T02:40:12.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:12.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:12.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:12.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:12.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:12.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:12.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:40:12.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_21 2026-03-24T02:40:12.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:12.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:12.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:40:12.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:12.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_21 ']' 2026-03-24T02:40:12.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-03-24T02:40:12.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-03-24T02:40:12.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:12.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:12.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:12.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:12.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:12.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:12.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:12.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:12.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:12.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:12.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:12.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:12.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-03-24T02:40:12.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-03-24T02:40:12.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:12.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:12.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:12.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_21 2026-03-24T02:40:12.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:12.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:12.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:12.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:12.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:12.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:12.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:12.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:12.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:12.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:12.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:12.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:12.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:12.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:12.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:12.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:12.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:12.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:12.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:12.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:12.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:12.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:12.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:12.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:12.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:12.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:12.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:12.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:12.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:12.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:12.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:12.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:12.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:12.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_21 2026-03-24T02:40:12.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:12.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-03-24T02:40:12.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:12.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:12.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:12.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-03-24T02:40:12.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:12.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:12.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:12.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:12.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-03-24T02:40:12.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_21.status 2026-03-24T02:40:12.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:12.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:12.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-03-24T02:40:12.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:13.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:13.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:13.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:13.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:13.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:13.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:13.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-03-24T02:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-03-24T02:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-03-24T02:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_21.status 2026-03-24T02:40:13.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:13.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:13.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-03-24T02:40:13.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:13.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:13.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:13.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:13.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:13.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:13.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:13.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:13.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:13.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:13.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:13.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:40:14.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_21 2026-03-24T02:40:14.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:14.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:14.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:40:14.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:14.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_21 ']' 2026-03-24T02:40:14.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-03-24T02:40:14.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-03-24T02:40:14.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:14.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:14.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:14.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:14.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:14.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:14.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:14.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:14.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:14.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:14.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:14.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:14.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:14.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:14.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:14.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:14.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:14.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:14.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:14.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:14.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:14.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:14.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:14.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-03-24T02:40:14.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-03-24T02:40:14.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:14.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:14.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_21 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:14.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:14.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:14.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:14.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:14.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_21 2026-03-24T02:40:14.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:14.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-03-24T02:40:14.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:14.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:14.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:14.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-03-24T02:40:14.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:14.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:14.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:14.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:14.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-03-24T02:40:14.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_21.status 2026-03-24T02:40:14.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:14.832 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:14.832 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-03-24T02:40:14.832 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:14.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:14.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:14.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:14.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:14.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:14.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:14.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:14.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-03-24T02:40:14.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:14.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:14.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:14.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-03-24T02:40:14.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:14.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:14.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:40:14.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:14.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-03-24T02:40:14.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_21.status 2026-03-24T02:40:14.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:14.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:14.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-03-24T02:40:14.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:14.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:14.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:14.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:14.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:14.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:14.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.status 2026-03-24T02:40:14.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:14.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:14.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:14.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_21 snap 2026-03-24T02:40:14.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:14.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:14.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:40:14.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:14.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:40:14.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:40:14.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:40:15.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_21@snap 2026-03-24T02:40:15.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_21': 2026-03-24T02:40:15.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:40:15.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:40:15.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:40:15.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c9d559f319 2026-03-24T02:40:15.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c9d559f319 2026-03-24T02:40:15.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:40:15.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:40:15.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:40:15.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:40:15.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:28:36 2026 2026-03-24T02:40:15.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:28:36 2026 2026-03-24T02:40:15.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:30:42 2026 2026-03-24T02:40:15.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:40:15.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c9d559f319 2026-03-24T02:40:15.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:40:15.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:40:15.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 43b3b561-7338-4109-92eb-2b42de532ec4 2026-03-24T02:40:15.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:40:15.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:15.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_21 snap 2026-03-24T02:40:15.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:15.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:40:15.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:15.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:40:15.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.export 2026-03-24T02:40:15.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_21.export 2026-03-24T02:40:15.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_21.export 2026-03-24T02:40:15.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_21@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.export 2026-03-24T02:40:16.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:40:16.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_21@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_21.export 2026-03-24T02:40:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:40:16.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_21.export 2026-03-24T02:40:16.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_21.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_21.export 2026-03-24T02:40:16.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:16.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:40:16.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_22 2026-03-24T02:40:16.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_22 snap 2026-03-24T02:40:16.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:16.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:16.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:40:16.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:16.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_22@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:40:17.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_22 2026-03-24T02:40:17.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:17.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:17.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:40:17.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_22 started 2026-03-24T02:40:17.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:17.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:17.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:40:17.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:40:17.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:40:17.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:17.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:40:18.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_22 started 2026-03-24T02:40:18.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:18.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:18.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:40:18.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:40:18.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:40:18.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:40:18.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_22 2026-03-24T02:40:18.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:40:18.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:40:18.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:18.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:18.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:40:18.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:40:18.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:18.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:40:18.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:18.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:40:18.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:18.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_22 2026-03-24T02:40:18.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:40:18.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:18.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:18.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:18.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:18.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:18.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:18.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:18.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:18.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:18.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:40:18.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:18.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:18.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:18.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:18.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_22 2026-03-24T02:40:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:40:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:18.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:18.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_22 2026-03-24T02:40:18.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:40:18.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:18.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:18.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:18.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:18.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:18.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:18.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:18.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:18.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:18.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:18.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:18.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:18.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:40:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:40:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:40:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:40:18.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:18.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:18.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:18.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:18.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_22 2026-03-24T02:40:18.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:18.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:40:18.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:18.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:18.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_22 2026-03-24T02:40:18.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:40:18.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:18.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:18.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:18.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:18.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:18.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:18.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:18.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:18.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:18.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:18.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:18.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:18.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:18.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:18.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:40:18.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:18.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:18.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:18.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:40:18.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:18.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:18.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:18.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:18.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:18.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:18.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:18.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:40:18.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:40:18.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_22 2026-03-24T02:40:18.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:18.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:40:18.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_22 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:40:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:18.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:18.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:18.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:40:18.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:18.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:18.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:18.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:40:18.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:40:18.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:40:18.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:18.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:18.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:18.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_22 2026-03-24T02:40:18.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:40:18.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:40:18.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:40:18.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:40:18.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_22 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_22 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:18.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:40:18.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_22 2026-03-24T02:40:18.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:18.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:18.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:40:18.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:18.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_22 ']' 2026-03-24T02:40:18.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-03-24T02:40:18.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-03-24T02:40:18.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:18.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:18.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:18.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:18.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:18.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:18.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:18.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:18.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:18.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:18.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:18.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:18.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:18.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:18.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:18.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:18.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:18.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:18.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:18.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:18.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:18.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:18.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:18.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:18.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:18.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-03-24T02:40:18.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:18.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:18.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:18.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:18.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:18.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:18.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:18.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:18.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:18.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:18.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:18.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:18.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:18.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:18.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:18.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:18.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:18.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:18.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:18.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:18.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:18.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:18.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:18.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:18.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:18.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:18.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:18.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:18.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:18.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:18.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:18.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:18.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:18.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:18.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:18.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:18.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:18.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:18.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_22 2026-03-24T02:40:18.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:18.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:18.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:18.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:18.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:18.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:18.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:18.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:18.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:18.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:18.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:18.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:18.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:18.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:18.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:18.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:18.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:18.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:18.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:18.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:18.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:18.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:18.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:18.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:18.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:18.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:18.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:18.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:18.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:18.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:18.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:18.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:18.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:18.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:19.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:19.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:19.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:19.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:19.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_22 2026-03-24T02:40:19.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:40:19.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:19.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:19.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:40:19.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:40:19.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:19.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:19.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:19.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:40:19.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:19.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:19.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:19.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:40:19.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:40:19.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:40:19.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:40:19.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:19.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:19.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:40:19.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:40:19.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:19.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:19.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:19.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:40:19.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:19.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:19.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:19.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:40:19.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:40:19.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:40:19.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:19.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:19.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:19.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:19.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:19.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-03-24T02:40:19.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:19.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:19.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:19.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-03-24T02:40:19.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:19.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:19.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:19.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:19.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-03-24T02:40:19.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_22.status 2026-03-24T02:40:19.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:19.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:19.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-03-24T02:40:19.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:19.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:40:19.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:40:19.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:19.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:19.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:19.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:19.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:40:19.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-03-24T02:40:19.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:19.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:19.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:19.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-03-24T02:40:19.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:19.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:19.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:19.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:19.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-03-24T02:40:19.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_22.status 2026-03-24T02:40:19.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:19.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:19.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:19.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-03-24T02:40:19.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:40:19.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:40:19.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:19.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:19.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:19.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:19.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:19.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:19.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:40:19.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:40:19.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:40:19.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:19.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:40:19.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:40:19.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:19.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:40:19.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:19.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:40:19.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:19.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:40:19.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:40:19.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:40:19.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:19.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:40:19.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:19.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:40:19.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:19.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:40:19.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:40:19.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:40:19.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:19.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:40:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_22 2026-03-24T02:40:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:40:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_22 ']' 2026-03-24T02:40:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-03-24T02:40:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-03-24T02:40:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:19.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:19.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-03-24T02:40:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:19.390 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:19.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:19.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:19.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:19.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:19.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:19.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:19.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:19.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:19.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:19.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:19.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:19.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:19.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:19.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:19.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:19.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:19.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:19.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:19.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_22 2026-03-24T02:40:19.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:19.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:19.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:19.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:19.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:19.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:19.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:19.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:19.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:19.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:19.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:19.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:19.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:19.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:19.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:19.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:19.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:19.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:19.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:19.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:19.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:19.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:19.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:19.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:19.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:19.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:19.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:19.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:19.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:19.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:19.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:19.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:19.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:19.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:19.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:19.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:19.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:19.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:19.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_22 2026-03-24T02:40:19.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:40:19.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:19.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:40:19.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:40:19.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:19.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:19.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:19.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:19.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:19.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-03-24T02:40:19.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:19.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:19.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:19.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-03-24T02:40:19.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:19.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:19.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:19.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:19.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-03-24T02:40:19.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_22.status 2026-03-24T02:40:19.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:19.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:19.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-03-24T02:40:19.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:19.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:40:19.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:40:19.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:19.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:19.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:19.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:19.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:40:19.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-03-24T02:40:19.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:19.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:19.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:19.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-03-24T02:40:19.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:19.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:19.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:19.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:19.693 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-03-24T02:40:19.693 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_22.status 2026-03-24T02:40:19.693 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:19.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:19.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-03-24T02:40:19.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:19.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:40:19.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:40:19.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:19.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:19.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:19.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:19.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:19.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:19.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:19.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:19.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:40:20.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_22 2026-03-24T02:40:20.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:20.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:20.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:40:20.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:20.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_22 ']' 2026-03-24T02:40:20.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-03-24T02:40:20.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-03-24T02:40:20.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:20.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:20.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:20.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:20.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:20.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:20.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:20.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:20.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:20.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:20.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:20.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:20.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:20.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:20.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:20.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:20.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:20.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:20.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:20.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:20.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:20.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:20.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:20.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:20.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:20.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:20.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:20.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:20.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:20.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:20.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:20.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:20.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:20.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:20.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:20.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:20.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:20.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:20.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:20.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_22 2026-03-24T02:40:20.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:20.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:20.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:20.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:20.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:20.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:20.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:20.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:20.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:20.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:20.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:20.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:20.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:20.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:20.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:20.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:20.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:20.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:20.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:20.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:20.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:20.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:20.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_22 2026-03-24T02:40:20.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:40:20.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:40:20.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:40:20.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:20.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:20.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:20.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:20.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-03-24T02:40:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-03-24T02:40:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-03-24T02:40:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_22.status 2026-03-24T02:40:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:20.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-03-24T02:40:20.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:20.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:40:20.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:40:20.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:20.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:20.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:20.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-03-24T02:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-03-24T02:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:20.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-03-24T02:40:20.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_22.status 2026-03-24T02:40:20.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:20.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:20.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-03-24T02:40:20.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:20.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:40:20.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:40:20.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:20.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:20.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:20.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:20.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:20.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:20.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:20.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:20.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:40:21.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_22 2026-03-24T02:40:21.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:21.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:21.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:40:21.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:21.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_22 ']' 2026-03-24T02:40:21.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-03-24T02:40:21.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-03-24T02:40:21.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:21.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:21.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:21.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:21.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:21.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:21.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:21.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:21.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:21.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:21.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:21.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:21.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:21.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:21.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:21.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:21.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:21.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:21.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:21.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:21.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:21.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:21.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:21.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:21.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:21.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:21.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-03-24T02:40:21.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:21.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:21.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:21.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:21.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:21.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:21.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:21.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:21.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:21.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:21.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:21.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:21.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:21.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:21.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:21.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:21.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:21.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:21.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:21.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:21.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:21.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:21.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:21.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:21.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:21.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:21.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:21.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:21.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:21.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:21.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:21.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:21.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:21.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:21.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:21.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:21.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:21.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:21.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_22 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:21.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:21.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:21.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:21.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:21.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:21.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:21.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:21.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:21.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_22 2026-03-24T02:40:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:40:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:40:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:40:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:40:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:40:21.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:40:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:40:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:40:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:40:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:40:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:40:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:40:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:40:21.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:40:21.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:21.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:21.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:21.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:21.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:21.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-03-24T02:40:21.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:21.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:21.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:21.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-03-24T02:40:21.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:21.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:21.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:21.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:21.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-03-24T02:40:21.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_22.status 2026-03-24T02:40:21.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:21.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:21.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-03-24T02:40:21.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:21.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:40:21.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:40:21.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:21.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:21.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:21.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:21.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:40:21.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-03-24T02:40:21.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:21.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:21.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:21.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-03-24T02:40:21.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:21.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:21.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:21.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:21.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-03-24T02:40:21.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_22.status 2026-03-24T02:40:21.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:21.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:21.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-03-24T02:40:21.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:21.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:40:21.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:40:21.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:21.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:21.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:21.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:21.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:21.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:21.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:21.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:21.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:40:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_22 2026-03-24T02:40:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:40:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_22 ']' 2026-03-24T02:40:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-03-24T02:40:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-03-24T02:40:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-03-24T02:40:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:23.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:23.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:23.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:23.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:23.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:23.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:23.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:23.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:23.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:23.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:23.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:23.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:23.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:23.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:23.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:23.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:23.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:23.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:23.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:23.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:23.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:23.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:23.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:23.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:23.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:23.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:23.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:23.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:23.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:23.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:23.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:23.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:23.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:23.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:23.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_22 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:23.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:23.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:23.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:23.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:23.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:23.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:23.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:23.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:23.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:23.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:23.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:23.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:23.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:23.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:23.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:23.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-03-24T02:40:23.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:23.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:23.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:23.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_22 2026-03-24T02:40:23.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:40:23.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:23.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:23.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:40:23.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:40:23.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:23.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:23.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:23.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:40:23.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:23.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:40:23.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:40:23.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:23.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:23.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:40:23.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_22 2026-03-24T02:40:23.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:23.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-03-24T02:40:23.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:23.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:23.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:23.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-03-24T02:40:23.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:23.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:23.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:23.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:23.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-03-24T02:40:23.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_22.status 2026-03-24T02:40:23.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:23.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:23.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-03-24T02:40:23.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:23.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:40:23.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:40:23.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:23.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:23.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:23.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:23.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:40:23.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-03-24T02:40:23.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:23.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:23.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:23.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-03-24T02:40:23.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:23.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:23.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:40:23.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:23.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-03-24T02:40:23.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_22.status 2026-03-24T02:40:23.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:23.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:23.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-03-24T02:40:23.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:23.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:40:23.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:40:23.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:23.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:23.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:23.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.status 2026-03-24T02:40:23.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:40:23.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:40:23.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:23.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_22 snap 2026-03-24T02:40:23.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:23.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:23.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:40:23.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:23.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:40:23.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:40:23.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:40:24.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_22@snap 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_22': 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10d116c6ad59 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10d116c6ad59 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:28:47 2026 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:28:47 2026 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:31:25 2026 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10d116c6ad59 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 7c6367d5-0206-4a8b-8be1-b152d19b9ac5 2026-03-24T02:40:24.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:40:24.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:24.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_22 snap 2026-03-24T02:40:24.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:24.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:40:24.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:24.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:40:24.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.export 2026-03-24T02:40:24.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_22.export 2026-03-24T02:40:24.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_22.export 2026-03-24T02:40:24.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_22@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.export 2026-03-24T02:40:24.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:40:24.827 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_22@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_22.export 2026-03-24T02:40:24.986 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:40:24.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_22.export 2026-03-24T02:40:25.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_22.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_22.export 2026-03-24T02:40:25.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:25.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:40:25.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_23 2026-03-24T02:40:25.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_23 snap 2026-03-24T02:40:25.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:25.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:25.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:40:25.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:25.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_23@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:40:25.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_23 2026-03-24T02:40:25.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:25.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:25.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:40:25.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_23 started 2026-03-24T02:40:25.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:25.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:25.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:40:25.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:40:25.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:40:25.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:25.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:40:26.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_23 started 2026-03-24T02:40:26.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:26.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:26.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:40:26.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:40:26.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:40:26.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:40:26.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_23 2026-03-24T02:40:26.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:40:26.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:40:26.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:26.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:26.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:40:26.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:40:26.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:26.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:40:26.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:26.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:40:26.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:26.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23 2026-03-24T02:40:26.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:40:26.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:26.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:26.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:26.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:26.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:26.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:26.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:26.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:26.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:26.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:26.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:26.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:26.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:26.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:26.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:40:26.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:26.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:26.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:26.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:40:26.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:26.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:26.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:26.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:26.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:26.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:26.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:26.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:40:26.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:40:26.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:26.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:26.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:26.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:26.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23 2026-03-24T02:40:26.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-03-24T02:40:26.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:40:26.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_23 2026-03-24T02:40:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:40:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:26.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:26.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:40:26.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:26.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:26.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:26.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:40:26.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:26.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:26.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:26.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:26.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:26.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:26.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:26.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:40:26.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:40:26.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:26.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:26.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:26.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:26.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_23 2026-03-24T02:40:26.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-03-24T02:40:26.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:40:26.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:26.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:26.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_23 2026-03-24T02:40:26.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:40:26.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:26.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:26.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:26.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:26.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:26.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:26.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:26.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:26.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:26.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:26.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:26.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:26.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:26.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:26.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:40:26.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:26.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:26.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:26.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:40:26.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:26.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:26.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:26.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:26.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:26.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:26.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:26.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:40:26.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:40:26.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:26.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:26.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:26.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:26.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_23 2026-03-24T02:40:26.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:40:26.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:40:26.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:40:26.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_23 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_23 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:40:26.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_23 2026-03-24T02:40:26.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:26.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:26.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:40:26.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:26.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_23 ']' 2026-03-24T02:40:26.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-03-24T02:40:26.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-03-24T02:40:26.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:26.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:26.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:26.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:26.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:26.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:26.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:26.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:26.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-03-24T02:40:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:26.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:26.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:26.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:26.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:26.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:26.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:26.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:26.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:26.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:26.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:26.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:26.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:26.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:26.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:26.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:26.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:26.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:26.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:26.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:26.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:26.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-03-24T02:40:26.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-03-24T02:40:26.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:26.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:26.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:26.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_23 2026-03-24T02:40:26.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:26.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:26.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:26.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:26.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:26.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:26.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:26.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:26.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:26.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:26.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:26.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:26.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:26.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:26.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:26.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:26.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:26.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:26.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:26.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:26.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:26.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:26.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:26.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:26.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:26.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:26.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:26.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:26.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:26.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:26.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:26.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:26.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:26.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_23 2026-03-24T02:40:26.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-03-24T02:40:26.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:26.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:26.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:26.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_23 2026-03-24T02:40:26.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:26.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:26.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:26.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:26.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:26.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:26.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:26.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:26.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:26.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:26.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:26.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:26.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:26.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:26.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:26.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:26.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:26.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:26.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:26.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:26.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:26.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:26.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:26.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:26.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:26.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:26.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:26.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:26.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:26.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:26.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:26.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:26.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:26.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_23 2026-03-24T02:40:26.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:26.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-03-24T02:40:26.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:26.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:26.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:26.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-03-24T02:40:26.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:26.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:26.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:26.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:26.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-03-24T02:40:26.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_23.status 2026-03-24T02:40:26.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:26.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:26.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-03-24T02:40:26.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:27.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:27.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:27.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:27.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:27.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:27.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:27.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:27.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-03-24T02:40:27.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:27.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:27.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:27.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-03-24T02:40:27.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:27.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:27.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:27.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:27.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-03-24T02:40:27.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_23.status 2026-03-24T02:40:27.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:27.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:27.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-03-24T02:40:27.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:27.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:27.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:27.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:27.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:27.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:27.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:27.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:27.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:27.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:40:27.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:40:27.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:40:27.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:27.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:27.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:40:27.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:27.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:40:27.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:27.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:40:27.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:27.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:40:27.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:27.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:40:27.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:27.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:40:27.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:27.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:40:27.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:27.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:40:27.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:40:27.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:40:27.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:27.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:40:27.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_23 2026-03-24T02:40:27.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:27.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:27.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:40:27.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:27.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_23 ']' 2026-03-24T02:40:27.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-03-24T02:40:27.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-03-24T02:40:27.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:27.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:27.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:27.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:27.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:27.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:27.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:27.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:27.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:27.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:27.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:27.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:27.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:27.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:27.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-03-24T02:40:27.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-03-24T02:40:27.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:27.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:27.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:27.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_23 2026-03-24T02:40:27.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:27.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:27.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:27.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:27.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:27.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:27.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:27.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:27.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:27.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:27.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:27.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:27.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:27.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_23 2026-03-24T02:40:27.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-03-24T02:40:27.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:27.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:27.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:27.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_23 2026-03-24T02:40:27.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:27.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:27.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:27.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:27.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:27.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:27.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_23 2026-03-24T02:40:27.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:27.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-03-24T02:40:27.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:27.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:27.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:27.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-03-24T02:40:27.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:27.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:27.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:27.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:27.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-03-24T02:40:27.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_23.status 2026-03-24T02:40:27.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:27.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:27.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-03-24T02:40:27.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:27.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:27.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:27.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:27.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:27.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:27.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:27.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:27.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-03-24T02:40:27.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:27.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:27.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:27.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-03-24T02:40:27.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:27.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:27.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:27.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:27.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-03-24T02:40:27.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_23.status 2026-03-24T02:40:27.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:27.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:27.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-03-24T02:40:27.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:27.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:27.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:27.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:27.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:27.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:27.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:27.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:27.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:27.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:27.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:27.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:40:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_23 2026-03-24T02:40:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:40:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_23 ']' 2026-03-24T02:40:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-03-24T02:40:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-03-24T02:40:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:27.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:27.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:27.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:27.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:27.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:27.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:27.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:27.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-03-24T02:40:28.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-03-24T02:40:28.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:28.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:28.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:28.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_23 2026-03-24T02:40:28.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:28.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:28.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:28.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:28.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:28.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:28.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:28.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:28.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:28.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:28.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_23 2026-03-24T02:40:28.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-03-24T02:40:28.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:28.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_23 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:28.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:28.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:28.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:28.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:28.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_23 2026-03-24T02:40:28.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:28.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-03-24T02:40:28.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:28.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:28.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:28.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-03-24T02:40:28.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:28.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:28.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:28.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:28.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-03-24T02:40:28.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_23.status 2026-03-24T02:40:28.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:28.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:28.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:28.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-03-24T02:40:28.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:28.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:28.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:28.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:28.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:28.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:28.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:28.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-03-24T02:40:28.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:28.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:28.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:28.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-03-24T02:40:28.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:28.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:28.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:28.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:28.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-03-24T02:40:28.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_23.status 2026-03-24T02:40:28.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-03-24T02:40:28.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:28.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:28.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:28.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:28.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:28.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:28.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:28.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:28.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:28.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:28.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:28.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:40:29.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_23 2026-03-24T02:40:29.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:29.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:29.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:40:29.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:29.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_23 ']' 2026-03-24T02:40:29.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-03-24T02:40:29.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-03-24T02:40:29.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:29.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:29.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:29.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:29.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:29.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:29.029 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:29.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:29.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:29.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:29.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:29.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:29.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:29.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:29.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:29.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-03-24T02:40:29.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-03-24T02:40:29.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:29.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_23 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:29.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:29.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:29.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:29.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:29.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:29.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:29.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:29.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:29.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:29.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:29.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:29.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:29.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:29.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:29.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:29.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:29.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:29.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_23 2026-03-24T02:40:29.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-03-24T02:40:29.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:29.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_23 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:29.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:29.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:29.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:29.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:29.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:29.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:29.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:29.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:29.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:29.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:29.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:29.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:29.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:29.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:29.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:29.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:29.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:29.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_23 2026-03-24T02:40:29.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:29.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-03-24T02:40:29.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:29.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:29.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:29.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-03-24T02:40:29.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:29.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:29.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:29.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:29.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-03-24T02:40:29.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_23.status 2026-03-24T02:40:29.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:29.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:29.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-03-24T02:40:29.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:29.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:29.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:29.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:29.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:29.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:29.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:29.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:29.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-03-24T02:40:29.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:29.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:29.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:29.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-03-24T02:40:29.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:29.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:29.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:29.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:29.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-03-24T02:40:29.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_23.status 2026-03-24T02:40:29.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:29.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:29.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-03-24T02:40:29.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:29.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:29.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:29.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:29.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:29.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:29.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:29.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:29.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:29.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:29.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:29.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:40:30.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_23 2026-03-24T02:40:30.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:30.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:30.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:40:30.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:30.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_23 ']' 2026-03-24T02:40:30.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-03-24T02:40:30.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-03-24T02:40:30.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:30.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:30.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:30.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:30.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:30.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:30.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:30.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:30.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-03-24T02:40:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:30.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:30.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:30.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:30.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:30.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:30.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-03-24T02:40:30.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-03-24T02:40:30.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:30.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:30.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:30.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_23 2026-03-24T02:40:30.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:31.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:31.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:31.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:31.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_23 2026-03-24T02:40:31.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-03-24T02:40:31.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:31.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:31.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:31.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_23 2026-03-24T02:40:31.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:31.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:31.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:31.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:31.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:31.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:31.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:31.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_23 2026-03-24T02:40:31.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:31.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-03-24T02:40:31.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:31.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:31.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:31.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-03-24T02:40:31.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:31.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:31.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:31.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:31.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-03-24T02:40:31.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_23.status 2026-03-24T02:40:31.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:31.150 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:31.150 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-03-24T02:40:31.150 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:31.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:31.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:31.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:31.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:31.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:31.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:31.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:31.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-03-24T02:40:31.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:31.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:31.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:31.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-03-24T02:40:31.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:31.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:31.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:40:31.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:31.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-03-24T02:40:31.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_23.status 2026-03-24T02:40:31.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:31.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:31.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-03-24T02:40:31.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:31.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:31.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:31.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:31.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:31.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:31.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.status 2026-03-24T02:40:31.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:31.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:31.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:31.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_23 snap 2026-03-24T02:40:31.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:31.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:31.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:40:31.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:31.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:40:31.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:40:31.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:40:32.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_23@snap 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_23': 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 116d1e1bfc98 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.116d1e1bfc98 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:28:59 2026 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:28:59 2026 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:31:17 2026 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 116d1e1bfc98 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 17a8da35-343b-4b4d-8bed-b592d7502654 2026-03-24T02:40:32.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:40:32.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:32.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_23 snap 2026-03-24T02:40:32.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:32.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:40:32.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:32.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:40:32.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.export 2026-03-24T02:40:32.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_23.export 2026-03-24T02:40:32.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_23.export 2026-03-24T02:40:32.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_23@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.export 2026-03-24T02:40:32.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:40:32.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_23@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_23.export 2026-03-24T02:40:32.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:40:32.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_23.export 2026-03-24T02:40:32.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_23.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_23.export 2026-03-24T02:40:32.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:32.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:40:32.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_24 2026-03-24T02:40:32.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_24 snap 2026-03-24T02:40:32.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:32.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:32.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:40:32.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:32.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_24@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:40:33.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_24 2026-03-24T02:40:33.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:33.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:33.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:40:33.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_24 started 2026-03-24T02:40:33.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:33.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:33.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:40:33.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:40:33.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:40:33.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:33.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:40:34.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_24 started 2026-03-24T02:40:34.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:34.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:34.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:40:34.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:40:34.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:40:34.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:40:34.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_24 2026-03-24T02:40:34.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:40:34.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:40:34.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:34.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:34.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:40:34.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:40:34.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:34.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:40:34.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:34.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:40:34.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:34.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_24 2026-03-24T02:40:34.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:40:34.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:34.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:34.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:40:34.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:34.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:34.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:34.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:34.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_24 2026-03-24T02:40:34.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-03-24T02:40:34.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:40:34.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_24 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:34.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:40:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:40:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:40:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:34.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:34.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:34.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:34.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_24 2026-03-24T02:40:34.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:40:34.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:40:34.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:40:34.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:40:34.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_24 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_24 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:34.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:40:34.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_24 2026-03-24T02:40:34.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:34.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:34.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:40:34.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:34.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_24 ']' 2026-03-24T02:40:34.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_24' 2026-03-24T02:40:34.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2026-03-24T02:40:34.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:34.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:34.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:34.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:34.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:34.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:34.712 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:34.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:34.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:34.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:34.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2026-03-24T02:40:34.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:34.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:34.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:34.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:34.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:34.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:34.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:34.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:34.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:34.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:34.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:34.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:34.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:34.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:34.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:34.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:34.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:34.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:34.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:34.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:34.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:34.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:34.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:34.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:34.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:34.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:34.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:34.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:34.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:34.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:34.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:34.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:34.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:34.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2026-03-24T02:40:34.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-03-24T02:40:34.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:34.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_24 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:34.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:34.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:34.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:34.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:34.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:34.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:34.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:34.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:34.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:34.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_24 2026-03-24T02:40:34.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:34.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_24 2026-03-24T02:40:34.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:34.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:34.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_24 2026-03-24T02:40:34.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_24 '' 2026-03-24T02:40:34.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:34.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:34.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_24 2026-03-24T02:40:34.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:34.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-03-24T02:40:34.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_24.status 2026-03-24T02:40:34.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:34.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:34.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-03-24T02:40:34.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:34.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:40:34.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:40:34.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:34.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:34.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:34.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:34.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:40:34.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2026-03-24T02:40:34.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:34.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:34.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_24 2026-03-24T02:40:34.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2026-03-24T02:40:34.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:34.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:34.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_24 2026-03-24T02:40:34.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:34.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-03-24T02:40:34.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_24.status 2026-03-24T02:40:34.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:34.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:34.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-03-24T02:40:34.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:34.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:40:34.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:40:34.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:34.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:34.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:34.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:34.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:34.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:34.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:40:34.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:40:34.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:40:34.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:34.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:40:34.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:40:34.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:34.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:40:34.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:34.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:40:34.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:34.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:40:34.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:40:34.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:40:34.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:34.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:40:34.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:34.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:40:34.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:34.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:40:34.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:40:34.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:40:34.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:34.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:40:35.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_24 2026-03-24T02:40:35.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:35.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:35.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:40:35.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:35.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_24 ']' 2026-03-24T02:40:35.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_24' 2026-03-24T02:40:35.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2026-03-24T02:40:35.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:35.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:35.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:35.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:35.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:35.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:35.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:35.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:35.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:35.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:35.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2026-03-24T02:40:35.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:35.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:35.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:35.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:35.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:35.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:35.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:35.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:35.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:35.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2026-03-24T02:40:35.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-03-24T02:40:35.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:35.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:35.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:35.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_24 2026-03-24T02:40:35.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:35.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:35.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:35.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:35.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:35.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:35.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:35.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:35.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:35.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:35.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:35.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:35.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:35.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:35.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:35.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:35.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:35.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:35.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:35.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:35.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:35.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:35.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:35.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:35.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:35.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:35.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:35.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:35.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:35.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:35.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:35.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:35.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:35.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_24 2026-03-24T02:40:35.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:35.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_24 2026-03-24T02:40:35.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:35.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:35.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_24 2026-03-24T02:40:35.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_24 '' 2026-03-24T02:40:35.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:35.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:35.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_24 2026-03-24T02:40:35.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:35.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-03-24T02:40:35.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_24.status 2026-03-24T02:40:35.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:35.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-03-24T02:40:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:35.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:40:35.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:40:35.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:35.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:35.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:35.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:35.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:40:35.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2026-03-24T02:40:35.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:35.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:35.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_24 2026-03-24T02:40:35.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2026-03-24T02:40:35.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:35.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:35.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_24 2026-03-24T02:40:35.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:35.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-03-24T02:40:35.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_24.status 2026-03-24T02:40:35.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:35.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:35.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-03-24T02:40:35.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:35.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:40:35.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:40:35.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:35.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:35.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:35.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:35.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:35.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:35.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:40:35.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:35.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:40:35.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_24 2026-03-24T02:40:35.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:35.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:35.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:40:35.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:35.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_24 ']' 2026-03-24T02:40:35.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_24' 2026-03-24T02:40:35.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2026-03-24T02:40:35.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:35.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:35.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:35.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:35.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:35.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:35.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:35.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:35.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:35.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:35.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2026-03-24T02:40:35.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-03-24T02:40:35.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:35.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:35.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:35.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_24 2026-03-24T02:40:35.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:35.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:35.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:35.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:35.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:35.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:35.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:35.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:35.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:35.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:35.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:35.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:35.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:35.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:35.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:35.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:35.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:35.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:35.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:35.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:35.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:35.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:35.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:35.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:35.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:35.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:35.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:35.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:35.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:35.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:35.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:35.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:35.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:35.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_24 2026-03-24T02:40:35.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:35.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_24 2026-03-24T02:40:35.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:35.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:35.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_24 2026-03-24T02:40:35.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_24 '' 2026-03-24T02:40:35.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:35.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:35.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_24 2026-03-24T02:40:35.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:35.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-03-24T02:40:35.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_24.status 2026-03-24T02:40:35.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:35.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:35.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-03-24T02:40:35.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:35.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:40:35.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:40:35.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:35.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:35.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:35.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:35.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:40:35.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2026-03-24T02:40:35.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:35.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:35.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_24 2026-03-24T02:40:35.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2026-03-24T02:40:35.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:35.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:35.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_24 2026-03-24T02:40:35.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:35.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-03-24T02:40:35.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_24.status 2026-03-24T02:40:35.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:35.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:35.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-03-24T02:40:35.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:36.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:40:36.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:40:36.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:36.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:36.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:36.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.status 2026-03-24T02:40:36.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:40:36.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:40:36.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:36.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_24 snap 2026-03-24T02:40:36.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:36.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:36.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:40:36.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:36.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:40:36.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:40:36.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:40:37.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_24@snap 2026-03-24T02:40:37.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_24': 2026-03-24T02:40:37.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:40:37.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:40:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:40:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 12091feec426 2026-03-24T02:40:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.12091feec426 2026-03-24T02:40:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:40:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:40:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:40:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:40:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:30:54 2026 2026-03-24T02:40:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:30:54 2026 2026-03-24T02:40:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:34:38 2026 2026-03-24T02:40:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:40:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 12091feec426 2026-03-24T02:40:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:40:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:40:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: c4b97ee8-4f98-4865-ac70-b188c34b4d9d 2026-03-24T02:40:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:40:37.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:37.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_24 snap 2026-03-24T02:40:37.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:37.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:40:37.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:37.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:40:37.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.export 2026-03-24T02:40:37.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_24.export 2026-03-24T02:40:37.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_24.export 2026-03-24T02:40:37.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_24@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.export 2026-03-24T02:40:37.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:40:37.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_24@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_24.export 2026-03-24T02:40:37.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:40:37.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_24.export 2026-03-24T02:40:37.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_24.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_24.export 2026-03-24T02:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_25 2026-03-24T02:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_25 snap 2026-03-24T02:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:37.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_25@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:40:38.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_25 2026-03-24T02:40:38.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:38.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:38.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:40:38.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_25 started 2026-03-24T02:40:38.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:38.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:38.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:40:38.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:40:38.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:40:38.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:38.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:40:39.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_25 started 2026-03-24T02:40:39.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:39.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:39.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:40:39.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:40:39.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:40:39.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:40:39.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_25 2026-03-24T02:40:39.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:40:39.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:39.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:40:39.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:39.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:40:39.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:40:39.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:39.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:40:39.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:39.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:40:39.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:39.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_25 2026-03-24T02:40:39.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:40:39.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:39.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:39.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:39.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:39.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:40:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:39.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:39.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:39.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:39.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_25 2026-03-24T02:40:39.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-03-24T02:40:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:40:39.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_25 2026-03-24T02:40:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:40:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:40:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:40:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:40:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:40:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:39.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:39.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:39.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:39.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_25 2026-03-24T02:40:39.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:40:39.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:40:39.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:40:39.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:40:39.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_25 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_25 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:39.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:40:39.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_25 2026-03-24T02:40:39.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:39.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:39.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:40:39.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:39.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_25 ']' 2026-03-24T02:40:39.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-03-24T02:40:39.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-03-24T02:40:39.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:39.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:39.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:39.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:39.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:39.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:39.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:39.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:39.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:39.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:39.731 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:39.731 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:39.731 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:39.731 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:39.731 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:39.731 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:39.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:39.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:39.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:39.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-03-24T02:40:39.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-03-24T02:40:39.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:39.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:39.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:39.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_25 2026-03-24T02:40:39.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:39.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:39.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:39.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:39.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:39.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:39.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:39.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:39.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:39.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:39.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:39.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:39.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:39.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:39.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:39.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:39.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:39.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:39.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_25 2026-03-24T02:40:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-03-24T02:40:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-03-24T02:40:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-03-24T02:40:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_25.status 2026-03-24T02:40:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:39.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:39.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:39.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-03-24T02:40:39.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:39.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:39.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:39.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:39.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:39.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:39.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-03-24T02:40:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-03-24T02:40:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-03-24T02:40:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_25.status 2026-03-24T02:40:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:39.912 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:39.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-03-24T02:40:39.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:39.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:39.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:40:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:39.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:40:39.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:39.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:40:39.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:39.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:40:39.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:39.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:40:39.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:39.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:40:39.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:39.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:40:39.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:39.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:40:39.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:40:39.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:40:39.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:39.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:40:40.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_25 2026-03-24T02:40:40.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:40.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:40.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:40:40.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:40.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_25 ']' 2026-03-24T02:40:40.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-03-24T02:40:40.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-03-24T02:40:40.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:40.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:40.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:40.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:40.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:40.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:40.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:40.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:40.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:40.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:40.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-03-24T02:40:40.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:40.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:40.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:40.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:40.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:40.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:40.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:40.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:40.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:40.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:40.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:40.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:40.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:40.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:40.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:40.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:40.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:40.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:40.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-03-24T02:40:40.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-03-24T02:40:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:40.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_25 2026-03-24T02:40:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:40.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:40.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:40.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:40.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:40.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:40.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:40.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:40.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:40.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:40.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:40.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:40.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:40.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:40.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:40.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:40.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:40.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:40.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:40.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:40.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_25 2026-03-24T02:40:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-03-24T02:40:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-03-24T02:40:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-03-24T02:40:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_25.status 2026-03-24T02:40:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-03-24T02:40:40.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:40.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:40.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:40.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:40.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:40.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:40.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:40.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:40.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-03-24T02:40:40.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:40.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:40.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:40.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-03-24T02:40:40.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:40.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:40.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:40.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:40.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-03-24T02:40:40.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_25.status 2026-03-24T02:40:40.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:40.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:40.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-03-24T02:40:40.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:40.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:40.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:40.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:40.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:40.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:40.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:40.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:40.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:40.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:40.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:40.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:40:40.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_25 2026-03-24T02:40:40.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:40.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:40.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:40:40.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:40.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_25 ']' 2026-03-24T02:40:40.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-03-24T02:40:40.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-03-24T02:40:40.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:40.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:40.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:40.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:40.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:40.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:40.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:40.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:40.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:40.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:40.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:40.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:40.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:40.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-03-24T02:40:40.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-03-24T02:40:40.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:40.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:40.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:40.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_25 2026-03-24T02:40:40.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:40.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:40.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:40.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:40.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:40.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:40.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:40.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:40.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:40.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:40.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_25 2026-03-24T02:40:40.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:40.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-03-24T02:40:40.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:40.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:40.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:40.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-03-24T02:40:40.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:40.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:40.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:40.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:40.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-03-24T02:40:40.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_25.status 2026-03-24T02:40:40.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:40.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:40.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-03-24T02:40:40.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:40.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:40.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:40.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:40.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:40.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:40.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:40.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:40.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-03-24T02:40:40.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:40.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:40.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:40.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-03-24T02:40:40.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:40.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:40.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:40.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:40.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-03-24T02:40:40.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_25.status 2026-03-24T02:40:40.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:40.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:40.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-03-24T02:40:40.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:40.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:40.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:40.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:40.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:40.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:40.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:41.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:41.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:41.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:41.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:41.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:40:41.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_25 2026-03-24T02:40:41.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:41.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:41.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:40:41.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:41.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_25 ']' 2026-03-24T02:40:41.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-03-24T02:40:41.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-03-24T02:40:41.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:41.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:41.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:41.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:41.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:41.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:41.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:41.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:41.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:41.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:41.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-03-24T02:40:41.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:41.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:41.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:41.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:41.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:41.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:41.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:41.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:41.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:41.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:41.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:41.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:41.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:41.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:41.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:41.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:41.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:41.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-03-24T02:40:41.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-03-24T02:40:41.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:41.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:41.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:41.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_25 2026-03-24T02:40:41.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:41.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:41.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:41.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:41.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:41.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:41.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_25 2026-03-24T02:40:41.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:41.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-03-24T02:40:41.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:41.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:41.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:41.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-03-24T02:40:41.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:41.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:41.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:41.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:41.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-03-24T02:40:41.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_25.status 2026-03-24T02:40:41.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:41.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:41.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-03-24T02:40:41.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:41.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:41.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:41.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:41.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:41.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:41.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:41.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:41.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-03-24T02:40:41.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:41.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:41.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:41.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-03-24T02:40:41.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:41.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:41.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:41.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:41.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-03-24T02:40:41.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_25.status 2026-03-24T02:40:41.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:41.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:41.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-03-24T02:40:41.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:42.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:42.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:42.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:42.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:42.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:42.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:42.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:42.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:42.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:40:42.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:42.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:40:43.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_25 2026-03-24T02:40:43.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:43.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:43.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:40:43.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:43.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_25 ']' 2026-03-24T02:40:43.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-03-24T02:40:43.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-03-24T02:40:43.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:43.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:43.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:43.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:43.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:43.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:43.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:43.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:43.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:43.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:43.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-03-24T02:40:43.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:43.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:43.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:43.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:43.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:43.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:43.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:43.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:43.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:43.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:43.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:43.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:43.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:43.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:43.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-03-24T02:40:43.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-03-24T02:40:43.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:43.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:43.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:43.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_25 2026-03-24T02:40:43.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:43.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:43.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:43.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:43.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:43.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:43.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:43.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:43.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:43.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:43.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:43.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:43.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:43.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:43.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:43.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:43.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:43.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:43.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:43.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:43.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:43.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:43.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:43.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:43.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:43.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:43.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:43.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:43.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:43.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:43.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:43.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:43.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:43.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_25 2026-03-24T02:40:43.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:43.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-03-24T02:40:43.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:43.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:43.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:43.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-03-24T02:40:43.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:43.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:43.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:43.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:43.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-03-24T02:40:43.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_25.status 2026-03-24T02:40:43.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:43.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:43.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-03-24T02:40:43.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:43.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:43.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:43.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-03-24T02:40:43.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-03-24T02:40:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:40:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-03-24T02:40:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_25.status 2026-03-24T02:40:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:43.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:43.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-03-24T02:40:43.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:43.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:40:43.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:40:43.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:43.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:43.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:43.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.status 2026-03-24T02:40:43.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:43.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:40:43.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:43.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_25 snap 2026-03-24T02:40:43.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:43.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:43.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:40:43.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:43.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:40:43.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:40:43.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:40:44.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_25@snap 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_25': 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c999631be5 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c999631be5 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:29:18 2026 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:29:18 2026 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:33:52 2026 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c999631be5 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 78e8e14a-d80c-40f9-8faa-8d35479f55a8 2026-03-24T02:40:44.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:40:44.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:44.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_25 snap 2026-03-24T02:40:44.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:44.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:40:44.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:44.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:40:44.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.export 2026-03-24T02:40:44.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_25.export 2026-03-24T02:40:44.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_25.export 2026-03-24T02:40:44.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_25@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.export 2026-03-24T02:40:45.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:40:45.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_25@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_25.export 2026-03-24T02:40:45.185 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:40:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_25.export 2026-03-24T02:40:45.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_25.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_25.export 2026-03-24T02:40:45.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:45.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:40:45.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_26 2026-03-24T02:40:45.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_26 snap 2026-03-24T02:40:45.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:45.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:45.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:40:45.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:45.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_26@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_26 2026-03-24T02:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_26 started 2026-03-24T02:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:45.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:40:46.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_26 started 2026-03-24T02:40:46.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:46.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:46.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:40:46.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:40:46.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:40:46.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:40:46.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_26 2026-03-24T02:40:46.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:40:46.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:40:46.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:46.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:46.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:40:46.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:40:46.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:46.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:40:46.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:46.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_26 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:46.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:46.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:46.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:46.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:46.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:46.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:40:46.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:40:46.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:46.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:46.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:46.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:46.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_26 2026-03-24T02:40:46.567 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-03-24T02:40:46.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:40:46.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_26 2026-03-24T02:40:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:40:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:40:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:40:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:40:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:40:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:46.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:46.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:46.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:46.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_26 2026-03-24T02:40:46.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-03-24T02:40:46.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:40:46.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:46.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:46.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_26 2026-03-24T02:40:46.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:40:46.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:46.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:46.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:46.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:46.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:46.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:46.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:46.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:46.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:46.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:46.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:46.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:46.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:40:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:40:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:40:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:40:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:46.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:46.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:46.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:46.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_26 2026-03-24T02:40:46.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:40:46.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:40:46.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:40:46.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_26 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_26 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:40:46.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_26 2026-03-24T02:40:46.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:46.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:46.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:40:46.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:46.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_26 ']' 2026-03-24T02:40:46.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-03-24T02:40:46.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-03-24T02:40:46.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:46.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:46.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:46.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:46.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:46.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:46.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:46.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:46.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:46.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:46.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-03-24T02:40:46.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:46.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:46.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:46.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:46.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:46.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:46.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:46.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:46.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:46.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:46.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:46.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-03-24T02:40:46.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-03-24T02:40:46.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:46.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:46.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:46.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_26 2026-03-24T02:40:46.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:46.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:46.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:46.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:46.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:46.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:46.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:47.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:47.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:47.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:47.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_26 2026-03-24T02:40:47.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-03-24T02:40:47.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:47.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:47.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:47.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_26 2026-03-24T02:40:47.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:47.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:47.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:47.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:47.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:47.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:47.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:47.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:47.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:47.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:47.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:47.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:47.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:47.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:47.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:47.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:47.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:47.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:47.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:47.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:47.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:47.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:47.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:47.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:47.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:47.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:47.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:47.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:47.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:47.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:47.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:47.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:47.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:47.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_26 2026-03-24T02:40:47.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:47.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-03-24T02:40:47.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:47.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:47.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:47.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-03-24T02:40:47.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:47.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:47.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:47.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:47.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-03-24T02:40:47.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_26.status 2026-03-24T02:40:47.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:47.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:47.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-03-24T02:40:47.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:47.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:47.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:47.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:47.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:47.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:47.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:47.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:47.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-03-24T02:40:47.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:47.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:47.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:47.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-03-24T02:40:47.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:47.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:47.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:47.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:47.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-03-24T02:40:47.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_26.status 2026-03-24T02:40:47.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:47.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:47.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-03-24T02:40:47.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:47.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:47.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:47.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:47.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:47.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:47.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:47.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:47.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:47.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:40:47.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:40:47.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:40:47.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:47.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:47.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:40:47.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:47.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:40:47.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:47.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:40:47.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:47.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:40:47.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:47.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:40:47.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:47.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:40:47.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:47.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:40:47.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:47.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:40:47.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:40:47.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:40:47.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:47.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:40:47.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_26 2026-03-24T02:40:47.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:47.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:47.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:40:47.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:47.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_26 ']' 2026-03-24T02:40:47.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-03-24T02:40:47.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-03-24T02:40:47.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:47.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:47.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:47.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:47.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:47.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:47.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:47.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:47.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:47.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:47.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-03-24T02:40:47.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:47.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:47.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:47.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:47.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:47.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:47.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:47.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:47.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:47.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:47.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:47.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:47.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:47.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:47.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:47.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:47.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:47.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:47.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:47.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:47.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:47.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:47.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:47.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:47.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:47.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:47.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:47.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:47.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:47.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:47.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:47.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:47.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:47.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-03-24T02:40:47.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-03-24T02:40:47.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:47.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_26 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:47.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:47.701 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_26 2026-03-24T02:40:47.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-03-24T02:40:47.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:47.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:47.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:47.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_26 2026-03-24T02:40:47.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:47.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:47.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:47.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:47.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:47.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:47.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:47.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:47.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_26 2026-03-24T02:40:47.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:47.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-03-24T02:40:47.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:47.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:47.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:47.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-03-24T02:40:47.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:47.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:47.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:47.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:47.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-03-24T02:40:47.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_26.status 2026-03-24T02:40:47.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:47.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:47.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-03-24T02:40:47.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:47.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:47.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:47.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-03-24T02:40:47.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:47.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:47.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:47.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-03-24T02:40:47.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:47.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:47.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:47.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:47.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-03-24T02:40:47.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_26.status 2026-03-24T02:40:47.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:47.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:47.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-03-24T02:40:47.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:47.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:47.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:47.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:47.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:47.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:47.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:47.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:47.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:47.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:47.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:47.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:40:48.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_26 2026-03-24T02:40:48.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:48.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:48.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:40:48.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:48.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_26 ']' 2026-03-24T02:40:48.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-03-24T02:40:48.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-03-24T02:40:48.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:48.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:48.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:48.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:48.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:48.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:48.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:48.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:48.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:48.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:48.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-03-24T02:40:48.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:48.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:48.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:48.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:48.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:48.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:48.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:48.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:48.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:48.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:48.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:48.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:48.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:48.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:48.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:48.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:48.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:48.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:48.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:48.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:48.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:48.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:48.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:48.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:48.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:48.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:48.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:48.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:48.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:48.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:48.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:48.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:48.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:48.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-03-24T02:40:48.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-03-24T02:40:48.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:48.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:48.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:48.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_26 2026-03-24T02:40:48.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:48.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:48.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:48.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:48.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:48.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:48.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:48.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:48.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:48.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:48.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:48.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:48.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:48.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:48.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:48.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:48.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:48.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:48.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:48.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:48.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:48.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:48.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:48.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:48.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:48.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:48.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:48.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:48.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:48.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:48.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:48.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:48.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:48.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_26 2026-03-24T02:40:48.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-03-24T02:40:48.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:48.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:48.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:48.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_26 2026-03-24T02:40:48.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:48.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:48.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:48.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:48.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:48.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:48.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:48.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:48.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:48.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:48.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:48.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:48.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:48.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:48.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:48.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:48.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:48.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:48.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:48.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:48.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:48.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:48.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:48.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:48.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:48.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:48.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:48.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:48.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:48.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:48.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:48.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:48.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:48.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_26 2026-03-24T02:40:48.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:48.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-03-24T02:40:48.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:48.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:48.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:48.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-03-24T02:40:48.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:48.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:48.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:48.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:48.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-03-24T02:40:48.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_26.status 2026-03-24T02:40:48.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:48.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:48.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-03-24T02:40:48.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:48.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:48.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:48.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:48.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:48.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:48.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:48.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:48.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-03-24T02:40:48.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:48.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:48.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:48.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-03-24T02:40:48.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:48.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:48.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:48.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:48.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-03-24T02:40:48.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_26.status 2026-03-24T02:40:48.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:48.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:48.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-03-24T02:40:48.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:48.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:48.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:48.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:48.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:48.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:48.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:48.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:48.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:48.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:48.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:40:49.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_26 2026-03-24T02:40:49.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:49.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:49.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:40:49.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:49.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_26 ']' 2026-03-24T02:40:49.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-03-24T02:40:49.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-03-24T02:40:49.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:49.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:49.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:49.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:49.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:49.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:49.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:49.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:49.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:49.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:49.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:49.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:49.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:49.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:49.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:49.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:49.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:49.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:49.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:49.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:49.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:49.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:49.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:49.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:49.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-03-24T02:40:49.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-03-24T02:40:49.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:49.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_26 2026-03-24T02:40:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:49.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:49.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:49.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:49.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:49.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:49.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:49.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:49.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:49.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:49.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:49.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:49.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:49.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:49.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:49.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:49.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:49.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:49.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:49.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_26 2026-03-24T02:40:49.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-03-24T02:40:49.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:49.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:49.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:49.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_26 2026-03-24T02:40:49.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:49.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:49.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:49.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:49.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:49.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:49.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:49.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:49.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:49.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:49.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:49.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:49.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:49.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_26 2026-03-24T02:40:49.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-03-24T02:40:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-03-24T02:40:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-03-24T02:40:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_26.status 2026-03-24T02:40:49.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:49.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:49.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-03-24T02:40:49.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:49.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:49.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:49.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:49.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:49.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:49.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:49.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:49.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-03-24T02:40:49.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:49.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:49.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:49.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-03-24T02:40:49.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:49.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:49.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:49.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:49.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-03-24T02:40:49.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_26.status 2026-03-24T02:40:49.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:49.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:49.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-03-24T02:40:49.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:49.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:49.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:49.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:49.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:49.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:49.690 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:49.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:49.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:49.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:49.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:49.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:40:51.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_26 2026-03-24T02:40:51.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:51.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:51.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:40:51.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:51.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_26 ']' 2026-03-24T02:40:51.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-03-24T02:40:51.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-03-24T02:40:51.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:51.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:51.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:51.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:51.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:51.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:51.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:51.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:51.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:51.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:51.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-03-24T02:40:51.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:51.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:51.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:51.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:51.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:51.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:51.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:51.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:51.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:51.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:51.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:51.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:51.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:51.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:51.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:51.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:51.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:51.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:51.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:51.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:51.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:51.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:51.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:51.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:51.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:51.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:51.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:51.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:51.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:51.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-03-24T02:40:51.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-03-24T02:40:51.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:51.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_26 2026-03-24T02:40:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:51.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:51.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:51.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:51.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:51.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:51.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:51.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:51.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:51.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:51.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:51.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:51.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:51.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:51.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:51.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:51.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:51.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:51.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:51.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:51.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_26 2026-03-24T02:40:51.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-03-24T02:40:51.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:51.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_26 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:51.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:51.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:40:51.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:40:51.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:51.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:51.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:51.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:40:51.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:40:51.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:51.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:51.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:40:51.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:40:51.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:40:51.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:51.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:51.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:40:51.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_26 2026-03-24T02:40:51.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:51.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-03-24T02:40:51.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:51.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:51.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:51.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-03-24T02:40:51.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:51.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:51.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:51.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:51.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-03-24T02:40:51.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_26.status 2026-03-24T02:40:51.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:51.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:51.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-03-24T02:40:51.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:51.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:51.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:51.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:51.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:51.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:51.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:51.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:51.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-03-24T02:40:51.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:51.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:51.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:51.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-03-24T02:40:51.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:51.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:51.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:40:51.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:51.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-03-24T02:40:51.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_26.status 2026-03-24T02:40:51.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:51.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:51.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-03-24T02:40:51.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:51.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:51.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:51.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:51.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:51.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:51.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.status 2026-03-24T02:40:51.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:51.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:51.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:51.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_26 snap 2026-03-24T02:40:51.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:51.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:51.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:40:51.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:51.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:40:51.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:40:51.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:40:52.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_26@snap 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_26': 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10d1e34828b1 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10d1e34828b1 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:29:21 2026 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:29:21 2026 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:34:58 2026 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10d1e34828b1 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 1e7e1f21-6b74-48d4-9119-089e1cc846ca 2026-03-24T02:40:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:40:52.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:52.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_26 snap 2026-03-24T02:40:52.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:52.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:40:52.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:52.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:40:52.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.export 2026-03-24T02:40:52.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_26.export 2026-03-24T02:40:52.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_26.export 2026-03-24T02:40:52.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_26@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.export 2026-03-24T02:40:52.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:40:52.794 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_26@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_26.export 2026-03-24T02:40:52.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:40:52.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_26.export 2026-03-24T02:40:53.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_26.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_26.export 2026-03-24T02:40:53.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:53.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:40:53.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_27 2026-03-24T02:40:53.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_27 snap 2026-03-24T02:40:53.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:53.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:53.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:40:53.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:53.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_27@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:40:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_27 2026-03-24T02:40:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:40:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_27 started 2026-03-24T02:40:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:40:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:40:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:40:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:40:54.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_27 started 2026-03-24T02:40:54.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:54.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:54.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:40:54.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:40:54.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:40:54.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:40:54.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_27 2026-03-24T02:40:54.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:40:54.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:54.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:54.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:40:54.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:40:54.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:54.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:40:54.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:40:54.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:54.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:40:54.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:54.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_27 2026-03-24T02:40:54.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:40:54.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:54.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:54.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:54.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:54.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:54.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:54.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:54.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:54.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:54.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:54.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:54.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:54.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:54.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:54.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:40:54.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:54.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:54.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:54.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:40:54.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:54.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:54.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:54.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:54.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:54.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:54.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:54.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:40:54.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:40:54.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:54.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:54.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:54.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:54.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_27 2026-03-24T02:40:54.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-03-24T02:40:54.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:40:54.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:54.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:54.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_27 2026-03-24T02:40:54.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:40:54.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:40:54.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:54.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:54.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:54.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:40:54.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:54.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:54.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:54.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:54.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:40:54.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:54.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:54.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:54.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:54.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:40:54.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:40:54.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:40:54.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:54.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:40:54.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:40:54.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:40:54.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:40:54.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:54.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:54.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:40:54.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:40:54.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:40:54.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:40:54.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:54.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:54.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:54.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:54.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_27 2026-03-24T02:40:54.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:40:54.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:40:54.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:40:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:40:54.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_27 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_27 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:54.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:40:54.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_27 2026-03-24T02:40:54.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:54.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:54.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:40:54.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:54.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_27 ']' 2026-03-24T02:40:54.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-03-24T02:40:54.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-03-24T02:40:54.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:54.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:54.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:54.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:54.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:54.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:54.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:54.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:54.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:54.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:54.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:54.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:54.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:54.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:54.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:54.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-03-24T02:40:54.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-03-24T02:40:54.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:54.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:54.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:54.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-03-24T02:40:54.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:54.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:54.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:54.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:54.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:54.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:54.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:54.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:54.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:54.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:54.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:54.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:54.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:54.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:54.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:54.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:54.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:54.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:54.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:54.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:54.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:54.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:54.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:54.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:54.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:54.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:54.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:54.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:54.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:54.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:54.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:54.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:54.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:54.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-03-24T02:40:55.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:55.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-03-24T02:40:55.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:55.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:55.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:55.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-03-24T02:40:55.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:55.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:55.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:55.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:55.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-03-24T02:40:55.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_27.status 2026-03-24T02:40:55.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:55.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:55.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-03-24T02:40:55.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:55.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:55.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:55.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:55.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:55.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:55.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:55.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:55.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-03-24T02:40:55.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:55.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:55.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:55.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-03-24T02:40:55.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:55.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:55.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:55.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:55.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-03-24T02:40:55.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_27.status 2026-03-24T02:40:55.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:55.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:55.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-03-24T02:40:55.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:55.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:55.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:55.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:55.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:55.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:55.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:55.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:55.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:55.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:40:55.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:40:55.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:40:55.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:55.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:55.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:40:55.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:55.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:40:55.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:55.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:40:55.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:55.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:40:55.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:55.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:40:55.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:55.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:40:55.095 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:55.095 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:40:55.095 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:40:55.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:40:55.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:40:55.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:40:55.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:55.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:40:55.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_27 2026-03-24T02:40:55.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:55.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:55.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:40:55.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:55.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_27 ']' 2026-03-24T02:40:55.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-03-24T02:40:55.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-03-24T02:40:55.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:55.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:55.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:55.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:55.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:55.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:55.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:55.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:55.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:55.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:55.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:55.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:55.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:55.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:55.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:55.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:55.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:55.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:55.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:55.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:55.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:55.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-03-24T02:40:55.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-03-24T02:40:55.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:55.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:55.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:55.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-03-24T02:40:55.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:55.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:55.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:55.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:55.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:55.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:55.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:55.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:55.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:55.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:55.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:55.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:55.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:55.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:55.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:55.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:55.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-03-24T02:40:55.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:55.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-03-24T02:40:55.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:55.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:55.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:55.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-03-24T02:40:55.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:55.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:55.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:55.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:55.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-03-24T02:40:55.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_27.status 2026-03-24T02:40:55.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:55.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:55.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-03-24T02:40:55.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:55.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:55.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:55.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:55.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:55.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:55.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:55.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:55.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-03-24T02:40:55.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:55.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:55.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:55.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-03-24T02:40:55.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:55.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:55.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:55.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:55.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-03-24T02:40:55.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_27.status 2026-03-24T02:40:55.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:55.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:55.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-03-24T02:40:55.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:55.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:55.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:55.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:55.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:55.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:55.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:55.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:55.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:55.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:55.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:55.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:40:55.597 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-03-24T02:40:55.597 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-03-24T02:40:55.597 INFO:tasks.rbd_mirror_thrash:waiting for 2.300557711790855 secs before reviving daemons 2026-03-24T02:40:55.598 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:40:55.593+0000 7f4dbaede640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 46106) UID: 1000 2026-03-24T02:40:55.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_27 2026-03-24T02:40:55.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:55.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:55.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:40:55.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:55.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_27 ']' 2026-03-24T02:40:55.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-03-24T02:40:55.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-03-24T02:40:55.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:55.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:55.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:55.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:55.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:55.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:55.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:55.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:55.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:55.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:55.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:55.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:55.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-03-24T02:40:56.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-03-24T02:40:56.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:56.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:56.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:56.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:56.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:56.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:56.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-03-24T02:40:56.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:56.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-03-24T02:40:56.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:56.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:56.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:56.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-03-24T02:40:56.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:56.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:56.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:56.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:56.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-03-24T02:40:56.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_27.status 2026-03-24T02:40:56.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:56.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:56.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-03-24T02:40:56.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:56.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:56.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:56.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:56.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:56.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:56.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:56.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:56.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-03-24T02:40:56.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:56.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:56.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:56.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-03-24T02:40:56.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:56.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:56.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:56.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:56.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-03-24T02:40:56.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_27.status 2026-03-24T02:40:56.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:56.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:56.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-03-24T02:40:56.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:56.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:56.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:56.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:56.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:56.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:56.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:56.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:56.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:56.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:56.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:56.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:40:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_27 2026-03-24T02:40:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:40:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_27 ']' 2026-03-24T02:40:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-03-24T02:40:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-03-24T02:40:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:56.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:56.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:56.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:56.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-03-24T02:40:56.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:56.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:56.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:56.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:56.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:56.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:56.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:56.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:56.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:56.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:56.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:56.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:56.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:56.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:56.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:56.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:56.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:56.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:56.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:56.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:56.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:56.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:56.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:56.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:56.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:56.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:56.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:56.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:56.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:56.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:56.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:56.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:56.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:56.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-03-24T02:40:57.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-03-24T02:40:57.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:57.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:57.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:57.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-03-24T02:40:57.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:57.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:57.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:57.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:57.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:57.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:57.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:57.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:57.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:57.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:57.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:57.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:57.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:57.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:57.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-03-24T02:40:57.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:57.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-03-24T02:40:57.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:57.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:57.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:57.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-03-24T02:40:57.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:57.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:57.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:57.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:57.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-03-24T02:40:57.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_27.status 2026-03-24T02:40:57.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:57.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:57.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-03-24T02:40:57.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:57.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:57.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:57.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:57.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:57.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:57.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:57.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:57.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-03-24T02:40:57.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:57.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:57.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:57.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-03-24T02:40:57.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:57.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:57.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:57.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:57.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-03-24T02:40:57.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_27.status 2026-03-24T02:40:57.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:57.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:57.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-03-24T02:40:57.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:57.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:57.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:57.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:57.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:57.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:40:57.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:57.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:57.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:57.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:40:57.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:40:57.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:40:57.898 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-03-24T02:40:57.898 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:40:57.898 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-03-24T02:40:57.898 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-03-24T02:40:57.898 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-03-24T02:40:57.900 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-03-24T02:40:57.900 INFO:tasks.rbd_mirror_thrash:waiting for 98.65728404210017 secs before thrashing 2026-03-24T02:40:58.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_27 2026-03-24T02:40:58.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:58.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:58.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:40:58.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:40:58.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_27 ']' 2026-03-24T02:40:58.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-03-24T02:40:58.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-03-24T02:40:58.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:40:58.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:58.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:58.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:40:58.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:40:58.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:40:58.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:40:58.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:40:58.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:40:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-03-24T02:40:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:40:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:40:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:40:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:40:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:40:58.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:40:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:40:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:40:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:40:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:40:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:40:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:40:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:40:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:40:58.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:58.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:58.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:40:58.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-03-24T02:40:58.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-03-24T02:40:58.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:40:58.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:40:58.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:40:58.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-03-24T02:40:58.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:40:58.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:40:58.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:40:58.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:40:58.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:40:58.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:40:58.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:40:58.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:40:58.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:40:58.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:58.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:40:58.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:58.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:40:58.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:58.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:40:58.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:40:58.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:40:58.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:40:58.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:40:58.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:40:58.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:40:58.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:40:58.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:40:58.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:40:58.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:40:58.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:40:58.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:40:58.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:40:58.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:40:58.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:40:58.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:58.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:58.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:40:58.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-03-24T02:40:58.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:58.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-03-24T02:40:58.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:58.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:58.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:58.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-03-24T02:40:58.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:58.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:58.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:58.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:40:58.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-03-24T02:40:58.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_27.status 2026-03-24T02:40:58.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:58.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:58.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-03-24T02:40:58.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:59.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:59.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:59.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:59.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:59.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:59.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:59.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:59.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-03-24T02:40:59.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:59.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:59.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:59.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-03-24T02:40:59.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:40:59.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:40:59.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:40:59.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:40:59.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-03-24T02:40:59.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_27.status 2026-03-24T02:40:59.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:40:59.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:59.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-03-24T02:40:59.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:59.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:40:59.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:40:59.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:40:59.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:59.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:40:59.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.status 2026-03-24T02:40:59.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:59.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:40:59.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:40:59.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_27 snap 2026-03-24T02:40:59.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:40:59.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:40:59.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:40:59.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:40:59.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:40:59.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:40:59.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:00.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_27@snap 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_27': 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 116d2ce63624 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.116d2ce63624 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:29:32 2026 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:29:32 2026 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:33:05 2026 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 116d2ce63624 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 8f55069e-2c11-406c-b7bc-f44992c3fcb1 2026-03-24T02:41:00.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:41:00.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:00.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_27 snap 2026-03-24T02:41:00.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:00.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:41:00.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:00.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:41:00.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.export 2026-03-24T02:41:00.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_27.export 2026-03-24T02:41:00.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_27.export 2026-03-24T02:41:00.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_27@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.export 2026-03-24T02:41:00.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:00.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_27@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_27.export 2026-03-24T02:41:00.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:00.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_27.export 2026-03-24T02:41:00.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_27.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_27.export 2026-03-24T02:41:00.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:00.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:41:00.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_28 2026-03-24T02:41:00.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_28 snap 2026-03-24T02:41:00.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:00.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:00.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:41:00.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:00.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_28@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:41:01.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_28 2026-03-24T02:41:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:41:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_28 started 2026-03-24T02:41:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:41:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:41:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_28 started 2026-03-24T02:41:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:41:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:41:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:41:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:41:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_28 2026-03-24T02:41:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:41:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:41:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:41:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:41:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:02.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:41:02.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:02.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:41:02.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:02.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_28 2026-03-24T02:41:02.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:41:02.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:02.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:41:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:41:02.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:02.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:02.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:02.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:02.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_28 2026-03-24T02:41:02.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-03-24T02:41:02.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:41:02.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_28 2026-03-24T02:41:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:41:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:41:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:41:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:41:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:41:02.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:02.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:02.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:02.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:02.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_28 2026-03-24T02:41:02.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:41:02.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:41:02.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:41:02.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_28 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_28 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:02.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:41:02.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_28 2026-03-24T02:41:02.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:02.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:02.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:41:02.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:02.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_28 ']' 2026-03-24T02:41:02.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_28' 2026-03-24T02:41:02.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2026-03-24T02:41:02.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:02.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:02.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:02.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:02.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:02.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:02.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:02.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:02.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:02.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:02.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:02.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:02.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:02.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:02.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:02.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:02.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:02.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:02.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:02.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2026-03-24T02:41:02.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-03-24T02:41:02.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:02.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:02.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:02.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2026-03-24T02:41:02.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:02.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:02.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:02.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:02.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:02.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:02.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:02.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:02.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:02.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:02.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:02.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:02.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:02.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:02.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:02.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:02.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:02.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:02.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:02.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:02.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:02.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:02.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:02.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:02.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:02.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:02.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:02.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:02.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:02.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:02.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:02.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:02.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:02.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2026-03-24T02:41:03.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:03.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_28 2026-03-24T02:41:03.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:03.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:03.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:03.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_28 '' 2026-03-24T02:41:03.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:03.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:03.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:03.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:03.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-03-24T02:41:03.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_28.status 2026-03-24T02:41:03.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:03.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:03.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:03.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-03-24T02:41:03.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:03.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:03.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:03.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:03.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:03.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:03.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:03.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2026-03-24T02:41:03.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:03.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:03.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:03.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2026-03-24T02:41:03.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:03.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:03.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:03.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:03.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-03-24T02:41:03.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_28.status 2026-03-24T02:41:03.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:03.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:03.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-03-24T02:41:03.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:03.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:03.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:03.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:03.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:03.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:03.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:41:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:41:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:41:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:41:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:03.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:41:03.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:03.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:41:03.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:03.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:41:03.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:03.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:41:03.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:03.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:41:03.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:03.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:41:03.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:03.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:41:03.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:41:03.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:41:03.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:03.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:41:03.140 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-03-24T02:41:03.140 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-03-24T02:41:03.140 INFO:tasks.rbd_mirror_thrash:waiting for 5.8081750759425805 secs before reviving daemons 2026-03-24T02:41:03.140 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:41:03.137+0000 7f7a5ce78640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 57415) UID: 1000 2026-03-24T02:41:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_28 2026-03-24T02:41:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:41:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_28 ']' 2026-03-24T02:41:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_28' 2026-03-24T02:41:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2026-03-24T02:41:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:03.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:03.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:03.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:03.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2026-03-24T02:41:03.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:03.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:03.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:03.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:03.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:03.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:03.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:03.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:03.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:03.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:03.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:03.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:03.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:03.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:03.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:03.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:03.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:03.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:03.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:03.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:03.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:03.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:03.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:03.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:03.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:03.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:03.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:03.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:03.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:03.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:03.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:03.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:03.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:03.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2026-03-24T02:41:03.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-03-24T02:41:03.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:03.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2026-03-24T02:41:03.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:03.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:03.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:03.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:03.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:03.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:03.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2026-03-24T02:41:03.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:03.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_28 2026-03-24T02:41:03.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:03.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:03.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:03.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_28 '' 2026-03-24T02:41:03.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:03.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:03.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:03.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:03.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-03-24T02:41:03.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_28.status 2026-03-24T02:41:03.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:03.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:03.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-03-24T02:41:03.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:03.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:03.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:03.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:03.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:03.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:03.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:03.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:03.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2026-03-24T02:41:03.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:03.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:03.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:03.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2026-03-24T02:41:03.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:03.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:03.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:03.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:03.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-03-24T02:41:03.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_28.status 2026-03-24T02:41:03.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:03.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:03.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-03-24T02:41:03.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:03.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:03.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:03.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:03.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:03.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:03.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:03.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:03.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:03.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:03.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:03.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:41:03.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_28 2026-03-24T02:41:03.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:03.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:03.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:41:03.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:03.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_28 ']' 2026-03-24T02:41:03.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_28' 2026-03-24T02:41:03.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2026-03-24T02:41:03.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:03.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:03.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:03.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:03.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:03.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:03.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:03.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:03.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:03.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:03.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2026-03-24T02:41:03.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:03.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:03.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:03.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:03.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:03.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:03.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:03.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:03.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:03.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:03.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:03.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:03.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:03.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:03.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:03.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:03.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:03.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:03.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:03.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:03.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:03.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:03.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:03.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:03.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:03.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:03.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:03.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:03.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:03.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:03.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:03.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:03.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:03.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2026-03-24T02:41:04.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:04.019 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:04.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:04.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:04.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:04.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:04.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2026-03-24T02:41:04.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:04.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_28 2026-03-24T02:41:04.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:04.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:04.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:04.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_28 '' 2026-03-24T02:41:04.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:04.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:04.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:04.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:04.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-03-24T02:41:04.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_28.status 2026-03-24T02:41:04.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:04.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:04.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-03-24T02:41:04.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:04.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:04.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:04.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:04.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:04.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:04.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:04.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:04.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2026-03-24T02:41:04.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:04.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:04.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:04.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2026-03-24T02:41:04.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:04.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:04.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:04.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:04.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-03-24T02:41:04.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_28.status 2026-03-24T02:41:04.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:04.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:04.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-03-24T02:41:04.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:04.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:04.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:04.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:04.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:04.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:04.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:04.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:04.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:04.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:04.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:04.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:41:05.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_28 2026-03-24T02:41:05.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:05.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:05.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:41:05.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:05.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_28 ']' 2026-03-24T02:41:05.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_28' 2026-03-24T02:41:05.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2026-03-24T02:41:05.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:05.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:05.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:05.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:05.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:05.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:05.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:05.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:05.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:05.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:05.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2026-03-24T02:41:05.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:05.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:05.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:05.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:05.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:05.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:05.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:05.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:05.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:05.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:05.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:05.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:05.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:05.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:05.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:05.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:05.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:05.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:05.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:05.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:05.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:05.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:05.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:05.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:05.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:05.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:05.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:05.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:05.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:05.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:05.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:05.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:05.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:05.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2026-03-24T02:41:05.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:05.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:05.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:05.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:05.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:05.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:05.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2026-03-24T02:41:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_28 2026-03-24T02:41:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_28 '' 2026-03-24T02:41:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:05.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-03-24T02:41:05.320 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_28.status 2026-03-24T02:41:05.320 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:05.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:05.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-03-24T02:41:05.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:05.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:05.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:05.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:05.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:05.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:05.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:05.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:05.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2026-03-24T02:41:05.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:05.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:05.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:05.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2026-03-24T02:41:05.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:05.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:05.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:05.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:05.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-03-24T02:41:05.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_28.status 2026-03-24T02:41:05.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:05.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:05.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-03-24T02:41:05.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:05.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:05.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:05.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:05.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:05.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:05.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:05.400 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:41:07.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_28 2026-03-24T02:41:07.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:07.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:07.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:41:07.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:07.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_28 ']' 2026-03-24T02:41:07.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_28' 2026-03-24T02:41:07.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2026-03-24T02:41:07.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:07.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:07.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:07.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:07.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:07.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:07.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:07.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:07.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:07.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:07.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:07.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:07.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:07.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:07.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:07.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:07.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:07.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:07.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:07.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:07.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:07.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:07.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:07.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:07.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2026-03-24T02:41:07.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-03-24T02:41:07.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:07.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2026-03-24T02:41:07.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:07.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:07.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:07.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:07.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:07.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:07.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:07.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:07.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:07.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:07.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:07.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:07.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:07.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:07.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:07.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:07.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:07.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:07.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:07.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:07.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:07.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:07.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:07.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:07.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:07.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:07.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:07.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:07.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:07.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:07.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:07.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:07.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:07.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2026-03-24T02:41:07.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:07.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_28 2026-03-24T02:41:07.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:07.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:07.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:07.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_28 '' 2026-03-24T02:41:07.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:07.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:07.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:07.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:07.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-03-24T02:41:07.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_28.status 2026-03-24T02:41:07.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:07.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:07.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-03-24T02:41:07.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:07.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:07.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:07.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:07.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:07.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:07.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:07.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:07.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2026-03-24T02:41:07.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:07.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:07.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:07.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2026-03-24T02:41:07.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:07.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:07.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:41:07.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-03-24T02:41:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_28.status 2026-03-24T02:41:07.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-03-24T02:41:07.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:07.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:07.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:07.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:07.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:07.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.status 2026-03-24T02:41:07.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:07.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:07.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:07.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_28 snap 2026-03-24T02:41:07.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:07.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:07.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:41:07.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:07.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:07.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:41:07.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:08.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_28@snap 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_28': 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 1209f5e5ea3 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.1209f5e5ea3 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:30:54 2026 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:30:54 2026 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:36:21 2026 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 1209f5e5ea3 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 8be4dac1-21bd-47c3-a3bc-61aa2ed6c651 2026-03-24T02:41:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:41:08.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:08.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_28 snap 2026-03-24T02:41:08.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:08.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:41:08.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:08.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:41:08.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.export 2026-03-24T02:41:08.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_28.export 2026-03-24T02:41:08.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_28.export 2026-03-24T02:41:08.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_28@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.export 2026-03-24T02:41:08.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:08.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_28@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_28.export 2026-03-24T02:41:08.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:08.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_28.export 2026-03-24T02:41:08.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_28.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_28.export 2026-03-24T02:41:08.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:41:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_29 2026-03-24T02:41:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_29 snap 2026-03-24T02:41:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:41:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_29@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:41:08.949 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-03-24T02:41:08.949 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:41:08.949 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-03-24T02:41:08.949 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-03-24T02:41:08.949 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-03-24T02:41:08.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-03-24T02:41:08.950 INFO:tasks.rbd_mirror_thrash:waiting for 65.26285433517582 secs before thrashing 2026-03-24T02:41:09.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_29 2026-03-24T02:41:09.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:09.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:09.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:41:09.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_29 started 2026-03-24T02:41:09.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:09.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:09.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:41:09.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:41:09.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:09.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:09.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:10.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_29 started 2026-03-24T02:41:10.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:10.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:10.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:41:10.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:41:10.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:41:10.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:41:10.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_29 2026-03-24T02:41:10.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:41:10.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:41:10.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:10.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:10.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:41:10.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:41:10.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:10.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:41:10.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:10.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:41:10.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:10.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_29 2026-03-24T02:41:10.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:41:10.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:10.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:10.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:10.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:10.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:10.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:10.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:10.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:10.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:41:10.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:41:10.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:10.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:10.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:10.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:10.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_29 2026-03-24T02:41:10.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-03-24T02:41:10.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:41:10.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:10.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:10.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_29 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:41:10.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:41:10.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:10.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:10.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:10.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:10.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_29 2026-03-24T02:41:10.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-03-24T02:41:10.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:41:10.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_29 2026-03-24T02:41:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:41:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:41:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:41:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:41:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:41:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:10.795 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:10.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:10.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:10.796 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_29 2026-03-24T02:41:10.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:41:10.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:41:10.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:41:10.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:41:10.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_29 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_29 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:41:11.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_29 2026-03-24T02:41:11.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:11.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:11.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:41:11.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:11.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_29 ']' 2026-03-24T02:41:11.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-03-24T02:41:11.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-03-24T02:41:11.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:11.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:11.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:11.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:11.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:11.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:11.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:11.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:11.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:11.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:11.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:11.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:11.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:11.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:11.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-03-24T02:41:11.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-03-24T02:41:11.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:11.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:11.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:11.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-03-24T02:41:11.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:11.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:11.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:11.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:11.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:11.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:11.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:11.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:11.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:11.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:11.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:11.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:11.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:11.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:11.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:11.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:11.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:11.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:11.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-03-24T02:41:11.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-03-24T02:41:11.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:11.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-03-24T02:41:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:11.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:11.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:11.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:11.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:11.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:11.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:11.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:11.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:11.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:11.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:11.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:11.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:11.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:11.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:11.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:11.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:11.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:11.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:11.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:11.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:11.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:11.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:11.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:11.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:11.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:11.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:11.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:11.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:11.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:11.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:11.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:11.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-03-24T02:41:11.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:11.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-03-24T02:41:11.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:11.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:11.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_29 2026-03-24T02:41:11.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-03-24T02:41:11.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:11.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:11.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_29 2026-03-24T02:41:11.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:11.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-03-24T02:41:11.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_29.status 2026-03-24T02:41:11.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:11.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_29.status 2026-03-24T02:41:11.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-03-24T02:41:11.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_29.status 2026-03-24T02:41:11.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:41:11.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:41:11.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:11.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:11.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:11.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_29.status 2026-03-24T02:41:11.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:11.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-03-24T02:41:11.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:11.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:11.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_29 2026-03-24T02:41:11.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-03-24T02:41:11.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:11.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:11.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_29 2026-03-24T02:41:11.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:11.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-03-24T02:41:11.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_29.status 2026-03-24T02:41:11.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:11.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_29.status 2026-03-24T02:41:11.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-03-24T02:41:11.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_29.status 2026-03-24T02:41:11.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:41:11.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:41:11.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:11.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:11.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:11.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_29.status 2026-03-24T02:41:11.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:41:11.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:41:11.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:41:11.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:41:11.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:41:11.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:41:11.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:11.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:41:11.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:11.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:41:11.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:41:11.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:41:11.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:11.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:41:11.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:11.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:41:11.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:11.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:41:11.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:41:11.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:41:11.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:11.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:41:11.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:41:11.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:41:11.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:11.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:41:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_29 2026-03-24T02:41:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:41:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_29 ']' 2026-03-24T02:41:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-03-24T02:41:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-03-24T02:41:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:11.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:11.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:11.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:11.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:11.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-03-24T02:41:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:11.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:11.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:11.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:11.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:11.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:11.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-03-24T02:41:11.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-03-24T02:41:11.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:11.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:11.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:11.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-03-24T02:41:11.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:11.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:11.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:11.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:11.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:11.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:11.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:11.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:11.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:11.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:11.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:11.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:11.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:11.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:11.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:11.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:11.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:11.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:11.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:11.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:11.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:11.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:11.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:11.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:11.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:11.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:11.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:11.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:11.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:11.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:11.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:11.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:11.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:11.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-03-24T02:41:11.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-03-24T02:41:11.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:11.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-03-24T02:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:11.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:11.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:11.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:11.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:11.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:11.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:11.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:11.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:11.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:11.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:11.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:11.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:11.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:11.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:11.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:11.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:11.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:11.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:11.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:11.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-03-24T02:41:11.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:11.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-03-24T02:41:11.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:11.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:11.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_29 2026-03-24T02:41:11.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-03-24T02:41:11.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:11.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:11.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_29 2026-03-24T02:41:11.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:11.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-03-24T02:41:11.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_29.status 2026-03-24T02:41:11.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_29.status 2026-03-24T02:41:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-03-24T02:41:11.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_29.status 2026-03-24T02:41:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:41:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:41:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:11.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_29.status 2026-03-24T02:41:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-03-24T02:41:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_29 2026-03-24T02:41:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-03-24T02:41:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_29 2026-03-24T02:41:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-03-24T02:41:11.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_29.status 2026-03-24T02:41:11.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:11.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_29.status 2026-03-24T02:41:11.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-03-24T02:41:11.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_29.status 2026-03-24T02:41:11.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:41:11.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:41:11.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:11.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:11.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:11.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_29.status 2026-03-24T02:41:11.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:11.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:11.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:11.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_29 snap 2026-03-24T02:41:11.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:11.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:11.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:41:11.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:11.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:11.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:41:11.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:12.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_29@snap 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_29': 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c96708c2f2 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c96708c2f2 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:29:53 2026 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:29:53 2026 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:36:06 2026 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c96708c2f2 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 3a4f296b-8a2c-440d-9a4c-2cb94425c87c 2026-03-24T02:41:12.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:41:12.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:12.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_29 snap 2026-03-24T02:41:12.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:12.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:41:12.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:12.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:41:12.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_29.export 2026-03-24T02:41:12.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_29.export 2026-03-24T02:41:12.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_29.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_29.export 2026-03-24T02:41:12.935 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_29@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_29.export 2026-03-24T02:41:13.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:13.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_29@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_29.export 2026-03-24T02:41:13.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:13.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_29.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_29.export 2026-03-24T02:41:13.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_29.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_29.export 2026-03-24T02:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_30 2026-03-24T02:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_30 snap 2026-03-24T02:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_30@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:41:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_30 2026-03-24T02:41:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:41:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_30 started 2026-03-24T02:41:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:41:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:41:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:13.751 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:14.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_30 started 2026-03-24T02:41:14.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:14.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:14.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:41:14.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:41:14.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:41:14.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:41:14.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_30 2026-03-24T02:41:14.753 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:41:14.753 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:14.753 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:14.753 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:41:14.753 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:41:14.753 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:14.753 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:41:14.753 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:41:14.753 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:14.753 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:41:14.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:14.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30 2026-03-24T02:41:14.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:41:14.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:14.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:14.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:14.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:14.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:14.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:14.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:14.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:14.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:14.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30 2026-03-24T02:41:14.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-03-24T02:41:14.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:41:14.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_30 2026-03-24T02:41:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:41:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:14.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:14.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:41:14.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:14.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:14.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:14.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:41:14.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:14.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:14.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:14.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:14.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:14.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:14.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:14.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:41:14.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:41:14.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:14.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:14.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:14.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:14.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_30 2026-03-24T02:41:14.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-03-24T02:41:14.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:41:14.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:14.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:14.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_30 2026-03-24T02:41:14.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:41:14.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:14.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:14.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:14.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:14.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:14.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:14.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:14.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:14.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:14.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:14.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:14.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:14.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:14.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:14.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:41:14.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:14.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:14.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:14.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:41:14.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:14.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:14.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:14.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:14.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:14.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:14.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:14.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:41:14.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:41:14.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:14.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:14.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:14.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:14.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_30 2026-03-24T02:41:14.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-03-24T02:41:14.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:41:14.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:14.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:14.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_30 2026-03-24T02:41:14.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:41:14.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:14.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:14.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:41:14.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:41:14.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:41:14.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:41:14.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:14.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:14.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:14.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_30 2026-03-24T02:41:15.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:41:15.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:41:15.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:41:15.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_30 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_30 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:15.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:41:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_30 2026-03-24T02:41:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:41:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_30 ']' 2026-03-24T02:41:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-03-24T02:41:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-03-24T02:41:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:15.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:15.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:15.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-03-24T02:41:15.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:15.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:15.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:15.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:15.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:15.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:15.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:15.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-03-24T02:41:15.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-03-24T02:41:15.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:15.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:15.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:15.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:15.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:15.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:15.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-03-24T02:41:15.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-03-24T02:41:15.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:15.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-03-24T02:41:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:15.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:15.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:15.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:15.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-03-24T02:41:15.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-03-24T02:41:15.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:15.502 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-03-24T02:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:15.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:41:15.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:15.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:15.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:15.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-03-24T02:41:15.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:15.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-03-24T02:41:15.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:15.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:15.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:41:15.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-03-24T02:41:15.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:15.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:15.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:41:15.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:15.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-03-24T02:41:15.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_30.status 2026-03-24T02:41:15.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:15.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:15.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-03-24T02:41:15.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:15.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:41:15.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:41:15.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:15.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:15.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:15.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:15.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:15.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-03-24T02:41:15.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:15.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:15.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:41:15.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-03-24T02:41:15.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:15.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:15.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:41:15.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:15.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-03-24T02:41:15.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_30.status 2026-03-24T02:41:15.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:15.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:15.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-03-24T02:41:15.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:15.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:41:15.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:41:15.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:15.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:15.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:15.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:15.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:41:15.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:41:15.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:41:15.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:41:15.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:41:15.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:41:15.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:15.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:41:15.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:15.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:41:15.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:41:15.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:41:15.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:15.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:41:15.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:15.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:41:15.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:15.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:41:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:41:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:41:15.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:41:15.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:41:15.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:41:15.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:15.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:41:15.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_30 2026-03-24T02:41:15.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:15.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:15.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:41:15.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:15.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_30 ']' 2026-03-24T02:41:15.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-03-24T02:41:15.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-03-24T02:41:15.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:15.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:15.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:15.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:15.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:15.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:15.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:15.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:15.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:15.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:15.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-03-24T02:41:15.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:15.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:15.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:15.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:15.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-03-24T02:41:15.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-03-24T02:41:15.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:15.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-03-24T02:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:15.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:15.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:15.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:15.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:15.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:15.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:15.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:15.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:15.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:15.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:15.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:15.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:15.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:15.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:15.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:15.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:15.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:15.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:15.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:15.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-03-24T02:41:16.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-03-24T02:41:16.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:16.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:16.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:16.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-03-24T02:41:16.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:16.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:16.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:16.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:16.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:16.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:16.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:16.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:16.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:16.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:16.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:16.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:16.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:16.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:16.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:16.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:16.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:16.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:16.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:16.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:16.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:16.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:16.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:16.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:16.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:16.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:16.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:16.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:16.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:16.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:16.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:16.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:16.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:16.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-03-24T02:41:16.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-03-24T02:41:16.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:16.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:16.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:16.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-03-24T02:41:16.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:41:16.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:16.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:16.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:41:16.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:41:16.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:16.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:16.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:16.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:41:16.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:16.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:16.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:16.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:41:16.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:16.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:41:16.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:41:16.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:16.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:16.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:41:16.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:41:16.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:16.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:16.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:16.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:16.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:16.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:16.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:16.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:41:16.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:16.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:41:16.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:16.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:16.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:16.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-03-24T02:41:16.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:16.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-03-24T02:41:16.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:16.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:16.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:41:16.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-03-24T02:41:16.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:16.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:16.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:41:16.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:16.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-03-24T02:41:16.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_30.status 2026-03-24T02:41:16.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:16.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:16.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:16.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-03-24T02:41:16.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:41:16.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:41:16.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:16.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:16.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:16.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:16.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:16.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-03-24T02:41:16.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:16.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:16.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:41:16.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-03-24T02:41:16.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:16.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:16.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:41:16.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:16.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-03-24T02:41:16.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_30.status 2026-03-24T02:41:16.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:16.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:16.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:16.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-03-24T02:41:16.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:41:16.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:41:16.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:16.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:16.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:16.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:16.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:41:16.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:41:16.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:41:16.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:16.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:41:16.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_30 2026-03-24T02:41:16.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:16.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:16.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:41:16.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:16.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_30 ']' 2026-03-24T02:41:16.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-03-24T02:41:16.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-03-24T02:41:16.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:16.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:16.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:16.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:16.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:16.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:16.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:16.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:16.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:16.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:16.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:16.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:16.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:16.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:16.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:16.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-03-24T02:41:16.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-03-24T02:41:16.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:16.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-03-24T02:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:16.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:16.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:16.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:16.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-03-24T02:41:17.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-03-24T02:41:17.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:17.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:17.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:17.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-03-24T02:41:17.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:17.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:17.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:17.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:17.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:17.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:17.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:17.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:17.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:17.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:17.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:17.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:17.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:17.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:17.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:17.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:17.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:17.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-03-24T02:41:17.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-03-24T02:41:17.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:17.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:41:17.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-03-24T02:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-03-24T02:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-03-24T02:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-03-24T02:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_30.status 2026-03-24T02:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:17.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:17.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-03-24T02:41:17.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:17.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:41:17.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:41:17.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:17.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:17.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:17.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:17.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:17.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-03-24T02:41:17.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:17.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:17.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:41:17.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-03-24T02:41:17.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:17.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:17.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:41:17.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:17.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-03-24T02:41:17.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_30.status 2026-03-24T02:41:17.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:17.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:17.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-03-24T02:41:17.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:17.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:41:17.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:41:17.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:17.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:17.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:17.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:17.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:41:17.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:41:17.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:41:17.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:17.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_30 2026-03-24T02:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_30 ']' 2026-03-24T02:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-03-24T02:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-03-24T02:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:18.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:18.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:18.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:18.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-03-24T02:41:18.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:18.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:18.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:18.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:18.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:18.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:18.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:18.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:18.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:18.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:18.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:18.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:18.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:18.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:18.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:18.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:18.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:18.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:18.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:18.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:18.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:18.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:18.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:18.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:18.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:18.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:18.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:18.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:18.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:18.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:18.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:18.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:18.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:18.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-03-24T02:41:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-03-24T02:41:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:18.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:18.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-03-24T02:41:18.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:18.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:18.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:18.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:18.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:18.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:18.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:18.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:18.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:18.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-03-24T02:41:18.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-03-24T02:41:18.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:18.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:18.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:18.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-03-24T02:41:18.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:18.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:18.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:18.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:18.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:18.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:18.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:18.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:18.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:18.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-03-24T02:41:18.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-03-24T02:41:18.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:18.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:18.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:18.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-03-24T02:41:18.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:41:18.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:18.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:18.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:41:18.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:41:18.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:18.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:18.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:18.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:41:18.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:18.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:18.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:18.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:41:18.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:18.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:41:18.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:41:18.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:18.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:18.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:41:18.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:41:18.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:18.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:18.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:18.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:18.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:18.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:18.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:18.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:41:18.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:18.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:41:18.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:18.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:18.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:18.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-03-24T02:41:18.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:18.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-03-24T02:41:18.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:18.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:18.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:41:18.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-03-24T02:41:18.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:18.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:18.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:41:18.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:18.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-03-24T02:41:18.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_30.status 2026-03-24T02:41:18.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:18.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:18.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-03-24T02:41:18.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:18.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:41:18.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:41:18.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:18.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:18.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:18.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:18.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:18.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-03-24T02:41:18.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:18.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:18.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:41:18.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-03-24T02:41:18.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:18.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:18.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:41:18.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:18.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-03-24T02:41:18.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_30.status 2026-03-24T02:41:18.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:18.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:18.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-03-24T02:41:18.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:18.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:41:18.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:41:18.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:18.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:18.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:18.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.status 2026-03-24T02:41:18.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:18.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:18.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:18.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_30 snap 2026-03-24T02:41:18.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:18.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:18.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:41:18.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:18.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:18.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:41:18.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:19.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_30@snap 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_30': 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10d12353f20d 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10d12353f20d 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:30:08 2026 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:30:08 2026 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:36:20 2026 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10d12353f20d 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: fa064077-42db-4a12-bd97-e2929e1ae111 2026-03-24T02:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:41:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_30 snap 2026-03-24T02:41:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:41:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:41:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.export 2026-03-24T02:41:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_30.export 2026-03-24T02:41:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_30.export 2026-03-24T02:41:19.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_30@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.export 2026-03-24T02:41:19.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:19.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_30@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_30.export 2026-03-24T02:41:19.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:19.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_30.export 2026-03-24T02:41:19.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_30.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_30.export 2026-03-24T02:41:19.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:19.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:41:19.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_31 2026-03-24T02:41:19.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_31 snap 2026-03-24T02:41:19.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:19.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:19.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:41:19.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:19.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_31@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:41:20.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_31 2026-03-24T02:41:20.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:20.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:20.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:41:20.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_31 started 2026-03-24T02:41:20.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:20.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:20.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:41:20.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:41:20.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:20.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:20.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:21.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_31 started 2026-03-24T02:41:21.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:21.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:21.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:41:21.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:41:21.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:41:21.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:41:21.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_31 2026-03-24T02:41:21.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:41:21.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:41:21.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:21.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:21.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:41:21.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:41:21.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:21.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:41:21.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_31 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:41:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:41:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:21.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:21.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:21.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:21.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_31 2026-03-24T02:41:21.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-03-24T02:41:21.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:41:21.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_31 2026-03-24T02:41:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:41:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:41:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:41:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:41:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:41:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:21.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:21.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:21.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:21.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_31 2026-03-24T02:41:21.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-03-24T02:41:21.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:41:21.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:21.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:21.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_31 2026-03-24T02:41:21.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:41:21.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:21.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:21.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:21.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:21.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:21.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:21.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:21.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:21.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:21.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:21.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:21.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:21.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:21.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:21.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:41:21.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:21.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:21.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:21.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:41:21.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:21.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:21.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:21.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:21.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:21.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:21.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:21.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:41:21.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:41:21.929 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:21.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:21.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:21.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:21.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_31 2026-03-24T02:41:22.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:41:22.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:41:22.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:41:22.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:41:22.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:41:22.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:41:22.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:41:22.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:22.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_31 2026-03-24T02:41:22.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:22.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:22.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:22.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:22.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:41:22.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:41:22.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_31 2026-03-24T02:41:22.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:22.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:22.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:22.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:22.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:41:22.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:41:22.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:41:22.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:41:22.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:22.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:41:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_31 2026-03-24T02:41:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:41:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_31 ']' 2026-03-24T02:41:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-03-24T02:41:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-03-24T02:41:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:22.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:22.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:22.218 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:22.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:22.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:22.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:22.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-03-24T02:41:22.286 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-03-24T02:41:22.286 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:22.286 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:22.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:22.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:22.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:22.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:22.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:22.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-03-24T02:41:22.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-03-24T02:41:22.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:22.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:22.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:22.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_31 2026-03-24T02:41:22.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:22.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:22.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:22.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:22.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:22.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:22.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:22.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:22.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:22.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:22.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:22.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:22.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:22.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:22.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:22.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:22.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:22.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_31 2026-03-24T02:41:22.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:22.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-03-24T02:41:22.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:22.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:22.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:22.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-03-24T02:41:22.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:22.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:22.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:22.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:22.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-03-24T02:41:22.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_31.status 2026-03-24T02:41:22.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:22.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:22.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-03-24T02:41:22.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:22.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:22.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:22.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:22.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:22.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:22.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:22.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:22.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-03-24T02:41:22.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:22.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:22.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:22.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-03-24T02:41:22.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:22.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:22.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:22.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:22.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-03-24T02:41:22.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_31.status 2026-03-24T02:41:22.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:22.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:22.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-03-24T02:41:22.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:22.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:22.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:22.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:22.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:22.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:22.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:22.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:22.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:22.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:41:22.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:41:22.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:41:22.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:22.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:22.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:41:22.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:22.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:41:22.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:22.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:41:22.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:22.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:41:22.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:22.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:41:22.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:22.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:41:22.532 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:22.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:41:22.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:22.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:41:22.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:41:22.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:41:22.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:22.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:41:22.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_31 2026-03-24T02:41:22.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:22.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:22.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:41:22.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:22.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_31 ']' 2026-03-24T02:41:22.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-03-24T02:41:22.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-03-24T02:41:22.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:22.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:22.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:22.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:22.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:22.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:22.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:22.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:22.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:22.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:22.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-03-24T02:41:22.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:22.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:22.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:22.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:22.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:22.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:22.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-03-24T02:41:22.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-03-24T02:41:22.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:22.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:22.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:22.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:22.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:22.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:22.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:22.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-03-24T02:41:22.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-03-24T02:41:22.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:22.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:22.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_31 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:22.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:22.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:22.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:22.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:22.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:22.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_31 2026-03-24T02:41:22.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:22.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-03-24T02:41:22.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:22.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:22.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:22.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-03-24T02:41:22.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:22.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:22.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:22.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:22.975 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-03-24T02:41:22.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_31.status 2026-03-24T02:41:22.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:22.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:22.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-03-24T02:41:22.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:23.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:23.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:23.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-03-24T02:41:23.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:23.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:23.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:23.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-03-24T02:41:23.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:23.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:23.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:23.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:23.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-03-24T02:41:23.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_31.status 2026-03-24T02:41:23.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:23.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:23.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-03-24T02:41:23.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:23.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:23.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:23.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:23.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:23.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:23.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:23.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:23.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:23.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:23.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:23.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:41:23.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_31 2026-03-24T02:41:23.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:23.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:23.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:41:23.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:23.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_31 ']' 2026-03-24T02:41:23.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-03-24T02:41:23.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-03-24T02:41:23.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:23.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:23.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:23.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:23.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:23.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:23.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:23.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:23.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:23.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:23.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-03-24T02:41:23.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:23.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:23.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:23.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:23.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:23.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:23.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:23.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:23.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:23.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:23.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:23.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:23.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:23.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:23.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:23.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:23.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:23.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:23.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:23.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:23.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:23.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:23.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:23.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:23.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:23.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:23.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:23.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:23.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:23.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:23.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:23.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-03-24T02:41:23.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-03-24T02:41:23.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:23.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:23.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:23.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-03-24T02:41:23.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:23.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:23.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:23.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:23.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:23.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:23.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:23.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:23.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:23.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:23.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:23.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:23.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:23.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:23.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:23.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:23.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:23.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:23.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:23.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:23.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:23.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:23.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:23.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:23.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:23.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:23.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:23.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:23.739 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:23.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:23.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:23.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:23.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:23.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-03-24T02:41:23.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-03-24T02:41:23.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:23.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:23.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:23.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_31 2026-03-24T02:41:23.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:23.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:23.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:23.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:23.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:23.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:23.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:23.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:23.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:23.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:23.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:23.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:23.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:23.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:23.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:23.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:23.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:23.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:23.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:23.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:23.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:23.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:23.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:23.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:23.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:23.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:23.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:23.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:23.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:23.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:23.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:23.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:23.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:23.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_31 2026-03-24T02:41:23.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:23.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-03-24T02:41:23.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:23.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:23.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:23.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-03-24T02:41:23.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:23.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:23.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:23.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:23.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-03-24T02:41:23.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_31.status 2026-03-24T02:41:23.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:23.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:23.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-03-24T02:41:23.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:23.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:23.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:23.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:23.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:23.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:23.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:23.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:23.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-03-24T02:41:23.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:23.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:23.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:23.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-03-24T02:41:23.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:23.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:23.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:23.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:23.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-03-24T02:41:23.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_31.status 2026-03-24T02:41:23.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:23.946 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:23.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-03-24T02:41:23.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:23.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:23.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:23.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:23.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:23.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:23.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:23.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:23.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:23.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:23.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:23.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:41:24.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_31 2026-03-24T02:41:24.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:24.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:24.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:41:24.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:24.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_31 ']' 2026-03-24T02:41:24.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-03-24T02:41:24.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-03-24T02:41:24.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:24.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:24.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:24.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:24.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:24.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:24.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:24.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:24.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:24.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:24.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-03-24T02:41:24.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:24.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:24.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:24.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:24.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:24.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:24.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:24.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:24.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:24.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:24.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:24.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:24.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:24.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:24.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:24.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:24.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:24.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:24.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-03-24T02:41:24.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-03-24T02:41:24.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:24.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:24.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:24.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-03-24T02:41:24.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:24.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:24.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:24.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:24.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:24.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:24.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:24.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:24.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:24.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:24.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:24.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:24.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:24.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:24.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:24.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:24.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:24.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:24.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:24.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:24.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:24.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:24.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:24.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:24.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:24.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:24.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:24.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:24.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:24.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:24.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:24.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:24.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:24.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-03-24T02:41:24.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-03-24T02:41:24.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:24.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_31 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:24.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:24.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:24.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:24.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:24.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:24.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:24.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:24.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:24.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:24.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:24.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:24.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_31 2026-03-24T02:41:25.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:25.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-03-24T02:41:25.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-03-24T02:41:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-03-24T02:41:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_31.status 2026-03-24T02:41:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:25.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:25.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-03-24T02:41:25.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:25.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:25.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:25.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:25.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:25.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:25.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:25.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:25.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-03-24T02:41:25.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:25.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:25.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:25.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-03-24T02:41:25.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:25.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:25.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:25.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:25.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-03-24T02:41:25.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_31.status 2026-03-24T02:41:25.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:25.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:25.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-03-24T02:41:25.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:25.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:25.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:25.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:25.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:25.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:25.080 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:25.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:25.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:25.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:25.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:25.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:41:26.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_31 2026-03-24T02:41:26.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:26.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:26.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:41:26.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:26.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_31 ']' 2026-03-24T02:41:26.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-03-24T02:41:26.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-03-24T02:41:26.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:26.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:26.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:26.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:26.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:26.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:26.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:26.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:26.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-03-24T02:41:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:26.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:26.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:26.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:26.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-03-24T02:41:26.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-03-24T02:41:26.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:26.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-03-24T02:41:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:26.760 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:26.761 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:26.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:26.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:26.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:26.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-03-24T02:41:26.833 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-03-24T02:41:26.833 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:26.833 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:26.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:26.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_31 2026-03-24T02:41:26.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:26.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:26.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:26.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:26.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:26.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:26.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:26.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:26.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:26.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:26.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:26.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:26.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_31 2026-03-24T02:41:26.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:26.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-03-24T02:41:26.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:26.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:26.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:26.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-03-24T02:41:26.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:26.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:26.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:26.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:26.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-03-24T02:41:26.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_31.status 2026-03-24T02:41:26.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:26.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:26.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:26.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-03-24T02:41:26.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:26.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:26.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:26.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:26.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:26.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:26.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:26.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-03-24T02:41:26.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:26.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:26.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:26.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-03-24T02:41:26.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:26.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:26.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:41:26.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:26.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-03-24T02:41:26.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_31.status 2026-03-24T02:41:26.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:26.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:26.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-03-24T02:41:26.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:26.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:26.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:26.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:26.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:26.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:26.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.status 2026-03-24T02:41:26.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:26.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:26.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:26.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_31 snap 2026-03-24T02:41:26.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:26.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:26.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:41:26.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:26.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:26.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:41:26.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:27.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_31@snap 2026-03-24T02:41:28.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_31': 2026-03-24T02:41:28.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:41:28.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:41:28.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:41:28.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 116d37a0eb9 2026-03-24T02:41:28.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.116d37a0eb9 2026-03-24T02:41:28.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:41:28.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:41:28.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:41:28.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:41:28.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:30:19 2026 2026-03-24T02:41:28.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:30:19 2026 2026-03-24T02:41:28.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:34:59 2026 2026-03-24T02:41:28.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:41:28.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 116d37a0eb9 2026-03-24T02:41:28.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:41:28.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:41:28.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: b05314b1-ea32-4f9b-9299-c6f9bc0a3e80 2026-03-24T02:41:28.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:41:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_31 snap 2026-03-24T02:41:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:41:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:41:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.export 2026-03-24T02:41:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_31.export 2026-03-24T02:41:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_31.export 2026-03-24T02:41:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_31@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.export 2026-03-24T02:41:28.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:28.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_31@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_31.export 2026-03-24T02:41:28.402 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:28.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_31.export 2026-03-24T02:41:28.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_31.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_31.export 2026-03-24T02:41:28.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:28.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:41:28.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_32 2026-03-24T02:41:28.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_32 snap 2026-03-24T02:41:28.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:28.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:28.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:41:28.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:28.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_32@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:41:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_32 2026-03-24T02:41:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:41:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_32 started 2026-03-24T02:41:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:41:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:41:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:29.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_32 started 2026-03-24T02:41:29.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:29.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:29.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:41:29.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:41:29.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:41:29.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:41:29.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:41:29.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_32 2026-03-24T02:41:29.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:41:29.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:29.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:29.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:41:29.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:41:29.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:29.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:41:29.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:29.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:41:29.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:29.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_32 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:29.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:29.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:29.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:41:29.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:29.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:29.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:29.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:29.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:29.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:29.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:29.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:41:29.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:41:29.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:29.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:29.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:29.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:29.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_32 2026-03-24T02:41:29.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:29.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:41:29.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:29.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:29.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_32 2026-03-24T02:41:29.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:41:29.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:29.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:29.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:29.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:29.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:29.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:29.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:29.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:29.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:41:29.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:41:29.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:29.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:29.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:29.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:29.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_32 2026-03-24T02:41:29.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:29.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:41:29.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_32 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:29.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:29.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:29.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:41:29.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:41:29.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:29.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:29.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:29.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:29.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_32 2026-03-24T02:41:30.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:30.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:41:30.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_32 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:41:30.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:41:30.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:30.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:30.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:41:30.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:41:30.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:30.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:30.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:30.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:30.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:30.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:30.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:30.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:41:30.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:41:30.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:41:30.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:30.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:30.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:30.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_32 2026-03-24T02:41:30.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:41:30.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:41:30.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:41:30.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_32 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_32 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:30.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:41:30.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_32 2026-03-24T02:41:30.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:30.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:30.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:41:30.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:30.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_32 ']' 2026-03-24T02:41:30.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-03-24T02:41:30.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-03-24T02:41:30.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:30.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:30.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:30.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:30.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:30.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:30.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:30.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:30.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:30.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:30.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-03-24T02:41:30.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:30.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:30.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:30.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:30.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:30.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:30.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:30.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:30.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:30.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:30.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:30.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:30.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:30.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:30.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:30.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:30.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:30.318 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:30.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:30.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:30.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:30.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:30.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-03-24T02:41:30.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:30.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:30.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:30.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:30.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:30.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:30.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:30.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:30.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:30.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:30.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:30.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:30.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:30.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:30.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:30.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:30.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:30.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:30.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:30.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:30.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:30.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:30.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:30.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:30.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:30.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:30.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:30.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:30.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:30.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:30.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:30.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:30.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:30.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:30.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:30.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:30.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:30.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:30.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_32 2026-03-24T02:41:30.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:30.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:30.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:30.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:30.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:30.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:30.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:30.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:30.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:30.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:30.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:30.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:30.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:30.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:30.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:30.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:30.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:30.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:30.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:30.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:30.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:30.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:30.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:30.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:30.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:30.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:30.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:30.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:30.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:30.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:30.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:30.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:30.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:30.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:30.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:30.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:30.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_32 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:41:30.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:30.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:41:30.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:30.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:30.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:30.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:30.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:30.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-03-24T02:41:30.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:30.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:30.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:30.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-03-24T02:41:30.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:30.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:30.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:30.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:30.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-03-24T02:41:30.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_32.status 2026-03-24T02:41:30.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:30.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:30.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-03-24T02:41:30.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:30.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:30.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:30.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:30.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:30.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:30.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:30.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:41:30.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-03-24T02:41:30.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:30.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:30.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:30.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-03-24T02:41:30.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:30.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:30.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:30.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:30.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-03-24T02:41:30.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_32.status 2026-03-24T02:41:30.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:30.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:30.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-03-24T02:41:30.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:30.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:30.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:30.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:30.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:30.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:30.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:30.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:30.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=4, entry_tid=1' -a 'object_number=5, tag_tid=4, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:30.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=3, entry_tid=1' '!=' '' 2026-03-24T02:41:30.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:41:30.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=3, entry_tid=1' = '' 2026-03-24T02:41:30.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:30.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:41:30.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:41:30.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:30.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=4 2026-03-24T02:41:30.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:30.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:41:30.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:30.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=3 2026-03-24T02:41:30.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:41:30.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:41:30.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:30.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:41:30.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:30.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:41:30.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:30.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=1 2026-03-24T02:41:30.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 4 = 3 -a 1 -le 1 2026-03-24T02:41:30.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:41:30.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:30.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:41:30.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_32 2026-03-24T02:41:30.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:30.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:30.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:41:30.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:30.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_32 ']' 2026-03-24T02:41:30.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-03-24T02:41:30.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-03-24T02:41:30.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:30.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:30.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:30.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:30.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:30.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:30.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:30.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:30.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:30.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:30.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-03-24T02:41:30.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:30.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:30.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:30.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:30.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:30.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:30.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:30.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:30.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:30.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:30.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:30.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:30.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:30.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:30.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:30.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:30.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:30.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:30.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:30.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:30.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:30.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:30.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:30.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:30.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:30.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:30.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:30.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:30.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:30.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:30.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:30.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:30.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:30.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:30.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:30.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:30.965 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:30.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:30.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:30.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:30.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:31.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:31.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:31.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:31.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:31.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_32 2026-03-24T02:41:31.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:31.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:31.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:31.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:31.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:31.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:31.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:31.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:31.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:31.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:31.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:31.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:31.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:31.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:31.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:31.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:31.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:31.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:31.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:31.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:31.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:31.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:31.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:31.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:31.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:31.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:31.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:31.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:31.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:31.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:31.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:31.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:31.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:31.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:31.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:31.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:31.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:31.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_32 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:31.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:41:31.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:31.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:31.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:31.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:31.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:31.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-03-24T02:41:31.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:31.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:31.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:31.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-03-24T02:41:31.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:31.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:31.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:31.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:31.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-03-24T02:41:31.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_32.status 2026-03-24T02:41:31.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:31.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:31.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-03-24T02:41:31.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:31.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:31.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:31.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:31.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:31.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:31.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:31.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:41:31.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-03-24T02:41:31.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:31.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:31.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:31.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-03-24T02:41:31.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:31.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:31.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:31.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:31.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-03-24T02:41:31.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_32.status 2026-03-24T02:41:31.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:31.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:31.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-03-24T02:41:31.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:31.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:31.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:31.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:31.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:31.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:31.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:31.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:31.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=4, entry_tid=1' -a 'object_number=5, tag_tid=4, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:31.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=3, entry_tid=1' '!=' 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:31.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:31.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:41:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_32 2026-03-24T02:41:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:41:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_32 ']' 2026-03-24T02:41:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-03-24T02:41:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-03-24T02:41:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:31.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:31.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:31.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:31.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-03-24T02:41:31.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:31.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:31.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:31.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:31.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:31.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:31.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:31.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:31.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:31.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:31.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:31.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:31.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:31.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:31.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:31.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:31.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:31.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:31.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:31.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:31.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:31.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:31.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:31.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:31.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:31.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:31.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:31.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:31.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:31.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:31.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:31.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:31.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:31.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:31.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:31.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_32 2026-03-24T02:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:31.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:31.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:31.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:31.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:31.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:31.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:31.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:31.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:31.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_32 2026-03-24T02:41:31.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:41:31.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:31.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:31.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:41:31.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:41:31.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:31.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:31.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:31.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:41:31.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:31.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:31.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:31.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:41:31.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:31.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:41:31.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:41:31.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:31.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:31.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:41:31.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:41:31.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:31.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:31.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:31.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:31.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:31.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:31.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:31.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:41:31.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:31.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:41:31.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:32.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:32.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-03-24T02:41:32.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:32.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:32.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:32.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-03-24T02:41:32.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:32.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:32.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:32.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:32.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-03-24T02:41:32.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_32.status 2026-03-24T02:41:32.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:32.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:32.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-03-24T02:41:32.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:32.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:32.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:32.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:32.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:32.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:32.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:32.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:41:32.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-03-24T02:41:32.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:32.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:32.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:32.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-03-24T02:41:32.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:32.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:32.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:32.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:32.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-03-24T02:41:32.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_32.status 2026-03-24T02:41:32.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:32.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:32.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-03-24T02:41:32.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:32.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:32.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:32.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:32.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:32.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:32.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:32.095 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:32.095 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=4, entry_tid=1' -a 'object_number=5, tag_tid=4, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:32.095 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=3, entry_tid=1' '!=' 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:32.095 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:32.095 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:41:32.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_32 2026-03-24T02:41:32.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:32.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:32.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:41:32.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:32.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_32 ']' 2026-03-24T02:41:32.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-03-24T02:41:32.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-03-24T02:41:32.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:32.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:32.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:32.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:32.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:32.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:32.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:32.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:32.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:32.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:32.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:32.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:32.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:32.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:32.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:32.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:32.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:32.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:32.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:32.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:32.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:32.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:32.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:32.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-03-24T02:41:32.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:32.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:32.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:32.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:32.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:32.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:32.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:32.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:32.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:32.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:32.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:32.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:32.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:32.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:32.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:32.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:32.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:32.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:32.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:32.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:32.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:32.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:32.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:32.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:32.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:32.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:32.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:32.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:32.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:32.980 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:32.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:32.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:32.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:32.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:33.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:33.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:33.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:33.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:33.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_32 2026-03-24T02:41:33.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:33.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:33.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:33.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:33.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:33.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:33.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:33.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:33.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:33.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:33.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:33.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:33.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:33.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:33.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:33.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:33.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:33.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:33.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:33.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:33.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:33.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:33.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:33.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:33.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:33.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:33.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:33.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:33.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:33.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:33.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:33.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:33.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:33.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:33.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:33.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:33.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_32 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:33.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:33.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:41:33.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:41:33.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:33.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:33.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:33.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:33.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:33.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:33.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:33.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:41:33.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:33.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:41:33.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:33.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:33.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:33.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-03-24T02:41:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-03-24T02:41:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:33.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-03-24T02:41:33.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_32.status 2026-03-24T02:41:33.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:33.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:33.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-03-24T02:41:33.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:33.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:33.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:33.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:33.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:33.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:33.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:33.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:41:33.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-03-24T02:41:33.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:33.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:33.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:33.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-03-24T02:41:33.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:33.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:33.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:33.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:33.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-03-24T02:41:33.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_32.status 2026-03-24T02:41:33.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:33.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-03-24T02:41:33.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:33.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:33.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:33.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:33.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:33.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:33.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:33.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:33.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=4, entry_tid=1' -a 'object_number=5, tag_tid=4, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:33.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=3, entry_tid=1' '!=' 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:33.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:33.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:41:34.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_32 2026-03-24T02:41:34.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:34.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:34.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:41:34.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:34.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_32 ']' 2026-03-24T02:41:34.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-03-24T02:41:34.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-03-24T02:41:34.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:34.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:34.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:34.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:34.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:34.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:34.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:34.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:34.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:34.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:34.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-03-24T02:41:34.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:34.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:34.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:34.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:34.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:34.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:34.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:34.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:34.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:34.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:34.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:34.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:34.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:34.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:34.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:34.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:34.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:34.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:34.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:34.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:34.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:34.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:34.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:34.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:34.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:34.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:34.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:34.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:34.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:34.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:34.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:34.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:34.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:34.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:34.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:34.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:34.981 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:34.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:34.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-03-24T02:41:34.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:34.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:34.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:34.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:34.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:34.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:34.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:34.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:34.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:34.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:34.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:34.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:34.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:34.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:34.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:34.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:34.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:34.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:34.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:34.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:34.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:34.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:34.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:34.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:34.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:34.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:34.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:34.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:34.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:34.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:34.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:34.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:34.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:34.998 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:35.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:35.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:35.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:35.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:35.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_32 2026-03-24T02:41:35.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:35.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:35.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:35.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:35.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:35.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:35.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:35.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:35.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:35.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:35.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:35.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:35.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:35.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:35.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:35.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:35.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:35.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:35.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:35.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:35.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:35.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:35.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:35.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:35.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:35.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:35.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:35.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:35.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:35.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:35.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:35.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:35.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:35.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:35.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:35.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:35.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:35.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:35.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_32 2026-03-24T02:41:35.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:41:35.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:35.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:35.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:41:35.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:41:35.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:35.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:35.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:35.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:35.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:41:35.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:35.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:35.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:35.154 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:35.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:35.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-03-24T02:41:35.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:35.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:35.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:35.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-03-24T02:41:35.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:35.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:35.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:35.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:35.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-03-24T02:41:35.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_32.status 2026-03-24T02:41:35.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-03-24T02:41:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:35.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:35.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:35.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:35.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:35.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:35.258 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:35.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:41:35.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-03-24T02:41:35.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:35.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:35.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:35.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-03-24T02:41:35.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:35.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:35.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:35.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:35.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-03-24T02:41:35.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_32.status 2026-03-24T02:41:35.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:35.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:35.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-03-24T02:41:35.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:35.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:35.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:35.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=4, entry_tid=1' -a 'object_number=5, tag_tid=4, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:35.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=3, entry_tid=1' '!=' 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:35.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:35.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:41:36.168 INFO:tasks.ceph.cluster2.osd.1.vm07.stderr:2026-03-24T02:41:36.165+0000 7f2dff19d640 -1 bdev(0x5602404d1400 /var/lib/ceph/osd/cluster2-1/block) aio_submit retries 1 2026-03-24T02:41:37.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_32 2026-03-24T02:41:37.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:37.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:37.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:41:37.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:37.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_32 ']' 2026-03-24T02:41:37.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-03-24T02:41:37.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-03-24T02:41:37.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:37.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:37.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:37.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:37.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:37.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:37.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:37.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:37.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-03-24T02:41:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:37.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:37.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:37.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:37.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:37.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:37.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:37.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:37.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:37.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:37.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:37.390 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:37.390 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:37.390 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:37.390 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:37.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:37.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:37.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_32 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:37.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:37.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-03-24T02:41:37.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:37.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:37.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:37.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_32 2026-03-24T02:41:37.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:41:37.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:37.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:37.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:41:37.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:41:37.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:37.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:37.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:37.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:41:37.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:37.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:37.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:37.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:41:37.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:37.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:41:37.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:41:37.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:37.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:37.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:41:37.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:41:37.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:37.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:37.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:37.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:37.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:37.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:37.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:37.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:41:37.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:37.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:41:37.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:37.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:37.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:37.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_32 2026-03-24T02:41:37.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:37.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-03-24T02:41:37.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:37.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:37.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:37.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-03-24T02:41:37.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:37.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:37.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:37.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:37.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-03-24T02:41:37.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_32.status 2026-03-24T02:41:37.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:37.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:37.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-03-24T02:41:37.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:37.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:37.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:37.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:37.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:37.651 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:37.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:37.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:41:37.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-03-24T02:41:37.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:37.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:37.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:37.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-03-24T02:41:37.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:37.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:37.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:41:37.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:37.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-03-24T02:41:37.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_32.status 2026-03-24T02:41:37.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:37.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:37.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-03-24T02:41:37.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:37.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:37.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:37.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:37.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:37.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6527], [object_number=6, tag_tid=1, entry_tid=6526]]], state=connected] 2026-03-24T02:41:37.693 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.status 2026-03-24T02:41:37.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:41:37.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=4, entry_tid=1' -a 'object_number=5, tag_tid=4, entry_tid=1' = 'object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:41:37.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:37.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_32 snap 2026-03-24T02:41:37.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:37.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:37.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:41:37.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:37.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:37.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:41:37.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:38.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_32@snap 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_32': 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 116db71cce7a 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.116db71cce7a 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:31:02 2026 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:31:02 2026 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:38:23 2026 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 116db71cce7a 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: d4d47d9c-e0e4-4f91-b35c-a45a2ad4bfe9 2026-03-24T02:41:38.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:41:38.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:38.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_32 snap 2026-03-24T02:41:38.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:38.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:41:38.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:38.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:41:38.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.export 2026-03-24T02:41:38.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_32.export 2026-03-24T02:41:38.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_32.export 2026-03-24T02:41:38.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_32@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.export 2026-03-24T02:41:38.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:38.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_32@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_32.export 2026-03-24T02:41:39.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:39.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_32.export 2026-03-24T02:41:39.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_32.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_32.export 2026-03-24T02:41:39.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:39.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:41:39.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_33 2026-03-24T02:41:39.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_33 snap 2026-03-24T02:41:39.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:39.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:39.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_33 2026-03-24T02:41:39.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:39.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_33@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:41:39.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_33 2026-03-24T02:41:39.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:39.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:39.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_33 2026-03-24T02:41:39.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_33 started 2026-03-24T02:41:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_33 2026-03-24T02:41:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:41:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:39.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:40.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_33 started 2026-03-24T02:41:40.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:40.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:40.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_33 2026-03-24T02:41:40.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:41:40.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:41:40.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:41:40.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_33 2026-03-24T02:41:40.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:41:40.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:41:40.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:40.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:40.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:41:40.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:41:40.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:40.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:41:40.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:40.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:41:40.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:40.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_33 2026-03-24T02:41:40.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:41:40.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:40.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:40.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:40.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:40.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:40.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:40.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:40.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:40.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:40.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:40.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:40.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:40.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:40.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:40.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:41:40.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:40.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:40.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:40.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:41:40.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:40.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:40.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:40.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:40.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:40.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:40.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:40.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:41:40.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:41:40.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:40.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_33 2026-03-24T02:41:40.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-03-24T02:41:40.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:41:40.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:40.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:40.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_33 2026-03-24T02:41:40.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:41:40.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:40.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:40.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:40.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:40.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:40.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:40.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:40.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:40.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:40.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:40.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:40.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:40.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:40.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:40.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:41:40.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:40.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:40.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:40.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:41:40.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:40.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:40.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:40.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:40.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:40.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:40.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:40.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:41:40.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:41:40.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:40.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:40.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:40.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:40.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_33 2026-03-24T02:41:41.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:41:41.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:41:41.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:41:41.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:41:41.044 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_33 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_33 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_33 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_33 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:41:41.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_33 2026-03-24T02:41:41.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:41.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:41.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_33 2026-03-24T02:41:41.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:41.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_33 ']' 2026-03-24T02:41:41.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-03-24T02:41:41.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-03-24T02:41:41.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:41.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:41.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:41.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:41.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:41.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:41.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:41.247 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:41.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:41.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:41.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:41.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:41.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:41.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-03-24T02:41:41.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-03-24T02:41:41.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:41.323 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:41.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:41.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-03-24T02:41:41.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:41.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:41.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:41.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:41.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:41.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:41.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:41.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:41.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:41.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:41.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:41.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:41.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:41.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:41.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:41.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:41.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:41.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:41.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:41.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:41.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:41.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:41.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:41.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:41.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:41.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:41.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:41.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:41.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:41.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:41.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:41.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:41.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:41.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-03-24T02:41:41.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:41.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-03-24T02:41:41.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:41.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:41.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_33 2026-03-24T02:41:41.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-03-24T02:41:41.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:41.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:41.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_33 2026-03-24T02:41:41.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:41.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-03-24T02:41:41.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_33.status 2026-03-24T02:41:41.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:41.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_33.status 2026-03-24T02:41:41.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-03-24T02:41:41.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_33.status 2026-03-24T02:41:41.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:41:41.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:41:41.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:41.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:41.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:41.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_33.status 2026-03-24T02:41:41.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:41.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-03-24T02:41:41.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:41.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:41.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_33 2026-03-24T02:41:41.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-03-24T02:41:41.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:41.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:41.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_33 2026-03-24T02:41:41.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:41.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-03-24T02:41:41.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_33.status 2026-03-24T02:41:41.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:41.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_33.status 2026-03-24T02:41:41.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-03-24T02:41:41.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_33.status 2026-03-24T02:41:41.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:41:41.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:41:41.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:41.490 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:41.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:41.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_33.status 2026-03-24T02:41:41.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:41.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:41.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:41.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_33 snap 2026-03-24T02:41:41.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:41.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:41.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_33 2026-03-24T02:41:41.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:41.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:41.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:41:41.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:42.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_33@snap 2026-03-24T02:41:42.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_33': 2026-03-24T02:41:42.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:41:42.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:41:42.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:41:42.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 120925ec8463 2026-03-24T02:41:42.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.120925ec8463 2026-03-24T02:41:42.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:41:42.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:41:42.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:41:42.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:41:42.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:30:34 2026 2026-03-24T02:41:42.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:30:34 2026 2026-03-24T02:41:42.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:37:26 2026 2026-03-24T02:41:42.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:41:42.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 120925ec8463 2026-03-24T02:41:42.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:41:42.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:41:42.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: f5118fd6-fb22-4687-9a18-bdc4a198974e 2026-03-24T02:41:42.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:41:42.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:42.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_33 snap 2026-03-24T02:41:42.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:42.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_33 2026-03-24T02:41:42.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:42.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:41:42.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_33.export 2026-03-24T02:41:42.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_33.export 2026-03-24T02:41:42.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_33.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_33.export 2026-03-24T02:41:42.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_33@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_33.export 2026-03-24T02:41:42.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:42.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_33@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_33.export 2026-03-24T02:41:42.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:42.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_33.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_33.export 2026-03-24T02:41:42.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_33.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_33.export 2026-03-24T02:41:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:41:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_34 2026-03-24T02:41:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_34 snap 2026-03-24T02:41:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:41:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_34@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:41:43.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_34 2026-03-24T02:41:43.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:43.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:43.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:41:43.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_34 started 2026-03-24T02:41:43.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:43.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:43.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:41:43.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:41:43.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:43.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:43.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:44.902 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_34 started 2026-03-24T02:41:44.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:44.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:44.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:41:44.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:41:44.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:41:44.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_34 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_34 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:44.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_34 2026-03-24T02:41:44.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:41:44.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:41:44.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:41:44.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:41:44.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_34 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_34 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:44.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:41:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_34 2026-03-24T02:41:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:41:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_34 ']' 2026-03-24T02:41:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-03-24T02:41:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-03-24T02:41:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:45.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:45.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:45.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:45.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:45.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-03-24T02:41:45.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:45.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:45.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:45.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:45.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:45.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:45.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:45.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:45.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:45.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:45.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:45.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:45.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:45.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:45.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:45.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:45.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:45.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-03-24T02:41:45.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:45.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-03-24T02:41:45.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:45.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:45.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:41:45.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-03-24T02:41:45.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:45.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:45.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:41:45.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:45.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-03-24T02:41:45.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_34.status 2026-03-24T02:41:45.286 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:45.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:45.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-03-24T02:41:45.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:45.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:41:45.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:41:45.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:45.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:45.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:45.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:45.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:41:45.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-03-24T02:41:45.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:45.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:45.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:41:45.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-03-24T02:41:45.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:45.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:45.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:41:45.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:45.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-03-24T02:41:45.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_34.status 2026-03-24T02:41:45.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:45.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:45.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-03-24T02:41:45.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:45.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:41:45.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:41:45.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:45.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:45.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:45.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:45.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:41:45.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:41:45.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:41:45.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:41:45.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:41:45.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:41:45.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:41:45.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:41:45.369 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:45.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:41:45.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:41:45.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:41:45.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:45.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:41:45.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:41:45.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:41:45.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:45.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:41:45.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:41:45.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:41:45.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:45.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:41:45.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:41:45.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:41:45.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:45.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:41:45.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_34 2026-03-24T02:41:45.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:45.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:45.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:41:45.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:45.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_34 ']' 2026-03-24T02:41:45.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-03-24T02:41:45.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-03-24T02:41:45.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:45.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:45.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:45.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:45.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:45.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:45.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:45.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:45.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:45.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:45.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:45.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:45.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:45.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:45.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:45.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:45.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:45.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:45.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:45.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:45.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:45.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:45.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:45.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:45.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:45.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:45.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:45.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-03-24T02:41:45.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:45.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-03-24T02:41:45.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:45.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:45.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:41:45.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-03-24T02:41:45.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:45.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:45.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:41:45.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:45.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-03-24T02:41:45.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_34.status 2026-03-24T02:41:45.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:45.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:45.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-03-24T02:41:45.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:45.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:41:45.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:41:45.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:45.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:45.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:45.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:45.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:41:45.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-03-24T02:41:45.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:45.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:45.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:41:45.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-03-24T02:41:45.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:45.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:45.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:41:45.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:45.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-03-24T02:41:45.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_34.status 2026-03-24T02:41:45.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:45.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:45.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-03-24T02:41:45.699 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:41:45.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:41:45.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:45.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:45.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:45.733 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:45.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:41:45.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:41:45.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:41:45.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:45.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:41:46.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_34 2026-03-24T02:41:46.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:46.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:46.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:41:46.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:46.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_34 ']' 2026-03-24T02:41:46.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-03-24T02:41:46.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-03-24T02:41:46.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:46.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:46.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:46.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:46.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:46.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:46.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:46.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:46.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:46.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:46.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:46.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:46.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:46.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:46.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:46.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:46.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:46.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:46.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:46.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:46.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-03-24T02:41:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-03-24T02:41:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:41:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-03-24T02:41:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:46.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:41:46.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:46.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-03-24T02:41:46.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_34.status 2026-03-24T02:41:46.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-03-24T02:41:46.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:46.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:41:46.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:41:46.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:46.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:46.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:46.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:46.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:41:46.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-03-24T02:41:46.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:46.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:46.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:41:46.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-03-24T02:41:46.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:46.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:46.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:41:46.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:46.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-03-24T02:41:46.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_34.status 2026-03-24T02:41:46.267 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:46.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:46.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-03-24T02:41:46.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:46.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:41:46.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:41:46.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:46.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:46.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:46.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:46.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:41:46.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:41:46.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:41:46.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:46.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_34 2026-03-24T02:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_34 ']' 2026-03-24T02:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-03-24T02:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-03-24T02:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:47.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:47.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-03-24T02:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:47.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:47.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:47.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:47.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:47.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-03-24T02:41:47.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:47.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-03-24T02:41:47.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:47.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:47.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:41:47.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-03-24T02:41:47.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:47.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:47.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:41:47.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:47.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-03-24T02:41:47.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:47.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_34.status 2026-03-24T02:41:47.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:47.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-03-24T02:41:47.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:47.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:41:47.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:41:47.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:47.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:47.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:47.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:47.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:41:47.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-03-24T02:41:47.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:47.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:47.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:41:47.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-03-24T02:41:47.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:47.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:47.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:41:47.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:47.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-03-24T02:41:47.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_34.status 2026-03-24T02:41:47.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:47.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:47.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-03-24T02:41:47.246 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:47.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:41:47.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:41:47.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:47.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:47.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:47.280 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.status 2026-03-24T02:41:47.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:41:47.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:41:47.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:47.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_34 snap 2026-03-24T02:41:47.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:47.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:47.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:41:47.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:47.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:47.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:41:47.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:48.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_34@snap 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_34': 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 12092c43f52a 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.12092c43f52a 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:30:47 2026 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:30:47 2026 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:35:35 2026 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 12092c43f52a 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 2d7b1a7f-d8c6-4e0a-9ec5-516114aa1857 2026-03-24T02:41:48.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:41:48.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:48.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_34 snap 2026-03-24T02:41:48.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:48.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:41:48.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:48.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:41:48.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.export 2026-03-24T02:41:48.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_34.export 2026-03-24T02:41:48.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_34.export 2026-03-24T02:41:48.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_34@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.export 2026-03-24T02:41:48.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:48.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_34@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_34.export 2026-03-24T02:41:48.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:48.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_34.export 2026-03-24T02:41:48.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_34.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_34.export 2026-03-24T02:41:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:41:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_35 2026-03-24T02:41:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_35 snap 2026-03-24T02:41:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:41:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:48.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_35@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:41:49.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_35 2026-03-24T02:41:49.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:49.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:49.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:41:49.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_35 started 2026-03-24T02:41:49.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:49.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:49.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:41:49.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:41:49.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:49.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:49.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:50.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_35 started 2026-03-24T02:41:50.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:50.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:50.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:41:50.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:41:50.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:41:50.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:41:50.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_35 2026-03-24T02:41:50.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:41:50.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:41:50.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:50.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:50.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:41:50.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:41:50.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:50.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:41:50.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:50.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:41:50.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:50.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_35 2026-03-24T02:41:50.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:41:50.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:50.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:50.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:50.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:50.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:50.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:50.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:50.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:50.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:50.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:50.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:50.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:50.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:50.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:50.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:41:50.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:50.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:50.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:50.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:41:50.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:50.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:50.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:50.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:50.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:50.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:50.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:50.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:41:50.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:41:50.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:50.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:50.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:50.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:50.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_35 2026-03-24T02:41:50.095 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-03-24T02:41:50.095 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:41:50.095 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:50.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:50.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_35 2026-03-24T02:41:50.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:41:50.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:50.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:50.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:50.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:50.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:50.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:50.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:50.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:50.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:50.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:41:50.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:41:50.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:50.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:50.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:50.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:50.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_35 2026-03-24T02:41:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-03-24T02:41:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:41:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:50.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:50.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_35 2026-03-24T02:41:50.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:41:50.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:50.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:50.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:50.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:50.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:50.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:50.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:50.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:50.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:50.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:50.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:50.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:50.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:50.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:50.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:41:50.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:50.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:50.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:50.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:41:50.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:50.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:50.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:50.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:50.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:50.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:50.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:50.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:41:50.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:41:50.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:50.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:50.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:50.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:50.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_35 2026-03-24T02:41:50.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-03-24T02:41:50.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:41:50.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:50.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:50.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_35 2026-03-24T02:41:50.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:41:50.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:50.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:50.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:41:50.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:41:50.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:50.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:50.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:50.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:50.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:50.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:50.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:50.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:41:50.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:50.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:41:50.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:41:50.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:50.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:50.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:41:50.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:41:50.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:50.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:50.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:50.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:50.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:50.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:50.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:50.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:41:50.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:41:50.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:41:50.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:50.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:50.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:50.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_35 2026-03-24T02:41:50.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:41:50.320 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:41:50.320 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:41:50.320 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:41:50.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:41:50.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:41:50.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:41:50.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:50.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_35 2026-03-24T02:41:50.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:50.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:50.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:50.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:50.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:41:50.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:41:50.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_35 2026-03-24T02:41:50.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:50.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:50.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:50.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:50.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:41:50.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:41:50.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:41:50.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:41:50.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:50.322 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:41:50.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_35 2026-03-24T02:41:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:41:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_35 ']' 2026-03-24T02:41:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-03-24T02:41:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-03-24T02:41:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:50.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:50.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-03-24T02:41:50.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:50.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:50.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:50.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:50.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:50.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:50.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:50.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:50.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:50.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:50.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:50.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:50.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:50.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:50.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:50.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:50.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:50.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:50.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:50.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:50.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:50.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:50.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:50.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:50.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:50.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:50.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:50.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:50.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:50.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:50.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:50.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:50.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:50.538 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-03-24T02:41:50.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-03-24T02:41:50.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:50.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:50.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:50.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_35 2026-03-24T02:41:50.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:50.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:50.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:50.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:50.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:50.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:50.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:50.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:50.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:50.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:50.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:50.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:50.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:50.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:50.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:50.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:50.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_35 2026-03-24T02:41:50.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-03-24T02:41:50.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:50.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_35 2026-03-24T02:41:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:50.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:50.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:50.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:50.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:50.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:50.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:50.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:50.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:50.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:50.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:50.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:50.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:50.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:50.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:50.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:50.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:50.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:50.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:50.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:50.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_35 2026-03-24T02:41:50.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-03-24T02:41:50.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:50.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_35 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:41:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:50.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:50.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:41:50.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:41:50.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:50.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:50.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:50.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:50.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:50.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:50.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:50.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:41:50.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:50.737 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:41:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_35 2026-03-24T02:41:50.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:50.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-03-24T02:41:50.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:50.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:50.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:41:50.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-03-24T02:41:50.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:50.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:50.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:41:50.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:50.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-03-24T02:41:50.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_35.status 2026-03-24T02:41:50.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:50.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:50.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-03-24T02:41:50.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:50.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:50.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:50.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:50.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:50.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:50.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:50.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:50.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-03-24T02:41:50.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:50.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:50.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:41:50.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-03-24T02:41:50.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:50.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:50.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:41:50.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:50.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-03-24T02:41:50.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_35.status 2026-03-24T02:41:50.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:50.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:50.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-03-24T02:41:50.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:50.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:50.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:50.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:50.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:50.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:50.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:50.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:50.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:50.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:41:50.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:41:50.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:41:50.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:50.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:50.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:41:50.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:50.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:41:50.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:50.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:41:50.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:50.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:41:50.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:50.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:41:50.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:50.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:41:50.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:50.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:41:50.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:41:50.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:41:50.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:41:50.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:41:50.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:50.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_35 2026-03-24T02:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_35 ']' 2026-03-24T02:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-03-24T02:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-03-24T02:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:51.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:51.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:51.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:51.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:51.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:51.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:51.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-03-24T02:41:51.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-03-24T02:41:51.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:51.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_35 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:51.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:51.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:51.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:51.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:51.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:51.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:51.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:51.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:51.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:51.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:51.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:51.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:51.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:51.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:51.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:51.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_35 2026-03-24T02:41:51.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-03-24T02:41:51.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:51.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:51.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:51.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_35 2026-03-24T02:41:51.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:51.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:51.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:51.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:51.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:51.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:51.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:51.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:51.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:51.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:51.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:51.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:51.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:51.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:51.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:51.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:51.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:51.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:51.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:51.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:51.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:51.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:51.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:51.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:51.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:51.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:51.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:51.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:51.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:51.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:51.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:51.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:51.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:51.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_35 2026-03-24T02:41:51.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-03-24T02:41:51.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:51.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_35 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:41:51.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:51.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:51.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:51.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:51.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:51.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:51.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:51.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:41:51.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:51.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:41:51.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:51.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:51.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:51.319 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_35 2026-03-24T02:41:51.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:51.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-03-24T02:41:51.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:51.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:51.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:41:51.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-03-24T02:41:51.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:51.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:51.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:41:51.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:51.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-03-24T02:41:51.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_35.status 2026-03-24T02:41:51.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:51.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:51.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-03-24T02:41:51.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:51.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:51.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:51.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:51.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:51.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:51.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:51.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:51.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-03-24T02:41:51.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:51.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:51.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:41:51.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-03-24T02:41:51.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:51.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:51.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:41:51.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:51.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-03-24T02:41:51.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_35.status 2026-03-24T02:41:51.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:51.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:51.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-03-24T02:41:51.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:51.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:51.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:51.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:51.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:51.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:51.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:51.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:51.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:51.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:51.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:51.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:41:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_35 2026-03-24T02:41:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:41:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_35 ']' 2026-03-24T02:41:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-03-24T02:41:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-03-24T02:41:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:51.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:51.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:51.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:51.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:51.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:51.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:51.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:51.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:51.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:51.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:51.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:51.869 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-03-24T02:41:51.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-03-24T02:41:51.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:51.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:51.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:51.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_35 2026-03-24T02:41:51.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:51.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:51.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:51.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:51.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:51.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:51.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:51.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:51.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:51.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:51.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:51.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:51.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:51.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_35 2026-03-24T02:41:51.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-03-24T02:41:51.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:51.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_35 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:51.996 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:52.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:52.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:52.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:52.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_35 2026-03-24T02:41:52.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-03-24T02:41:52.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:52.065 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:52.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:52.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_35 2026-03-24T02:41:52.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:41:52.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:52.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:52.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:41:52.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:41:52.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:52.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:52.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:52.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:41:52.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:52.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:52.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:52.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:41:52.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:52.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:41:52.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:41:52.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:52.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:52.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:41:52.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:41:52.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:52.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:52.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:52.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:52.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:52.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:52.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:52.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:41:52.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:52.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:41:52.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:52.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:52.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:52.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_35 2026-03-24T02:41:52.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:52.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-03-24T02:41:52.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:52.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:52.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:41:52.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-03-24T02:41:52.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:52.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:52.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:41:52.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:52.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-03-24T02:41:52.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_35.status 2026-03-24T02:41:52.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:52.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:52.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-03-24T02:41:52.146 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:52.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:52.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:52.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:52.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:52.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:52.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:52.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:52.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-03-24T02:41:52.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:52.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:52.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:41:52.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-03-24T02:41:52.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:52.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:52.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:41:52.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:52.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-03-24T02:41:52.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_35.status 2026-03-24T02:41:52.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:52.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:52.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-03-24T02:41:52.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:52.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:52.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:52.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:41:52.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:52.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:52.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:52.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:41:52.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:52.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:41:53.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_35 2026-03-24T02:41:53.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:53.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:53.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:41:53.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:53.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_35 ']' 2026-03-24T02:41:53.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-03-24T02:41:53.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-03-24T02:41:53.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:53.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:53.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:53.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:53.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:53.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:53.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:53.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:53.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-03-24T02:41:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:53.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:53.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:53.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:53.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:53.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:53.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-03-24T02:41:53.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-03-24T02:41:53.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:53.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:53.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:53.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_35 2026-03-24T02:41:53.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:53.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:53.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:53.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:53.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:53.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:53.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:53.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:53.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:53.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:53.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:53.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:53.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:53.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:53.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:53.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:53.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:53.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:53.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:53.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:53.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:53.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:53.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:53.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:53.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:53.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:53.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:53.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:53.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:53.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:53.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:53.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:53.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:53.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_35 2026-03-24T02:41:53.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-03-24T02:41:53.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:53.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:53.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_35 2026-03-24T02:41:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:41:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:41:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:41:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:41:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:41:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:41:53.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:41:53.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:53.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:53.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:41:53.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:41:53.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:53.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:53.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:53.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:41:53.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:41:53.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:53.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:53.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:41:53.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:41:53.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:41:53.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:53.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:53.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:41:53.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_35 2026-03-24T02:41:53.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-03-24T02:41:53.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:53.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:53.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:53.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_35 2026-03-24T02:41:53.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:41:53.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:53.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:53.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:41:53.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:41:53.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:53.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:53.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:53.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:41:53.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:53.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:53.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:53.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:41:53.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:41:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:41:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:41:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:41:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:41:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:41:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:41:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:41:53.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:41:53.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:53.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:53.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:41:53.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_35 2026-03-24T02:41:53.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:53.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-03-24T02:41:53.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:53.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:53.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:41:53.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-03-24T02:41:53.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:53.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:53.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:41:53.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:53.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-03-24T02:41:53.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_35.status 2026-03-24T02:41:53.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:53.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:53.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-03-24T02:41:53.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:53.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:53.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:53.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:53.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:53.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:53.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:53.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-03-24T02:41:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:41:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-03-24T02:41:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:41:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-03-24T02:41:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_35.status 2026-03-24T02:41:53.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:53.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:53.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:53.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-03-24T02:41:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:41:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:41:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:53.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.status 2026-03-24T02:41:53.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:53.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:41:53.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:53.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_35 snap 2026-03-24T02:41:53.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:53.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:53.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:41:53.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:53.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:53.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:41:53.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:54.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_35@snap 2026-03-24T02:41:54.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_35': 2026-03-24T02:41:54.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:41:54.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:41:54.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:41:54.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 1209333d4690 2026-03-24T02:41:54.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.1209333d4690 2026-03-24T02:41:54.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:41:54.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:41:54.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:41:54.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:41:54.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:30:50 2026 2026-03-24T02:41:54.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:30:50 2026 2026-03-24T02:41:54.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:36:37 2026 2026-03-24T02:41:54.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:41:54.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 1209333d4690 2026-03-24T02:41:54.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:41:54.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:41:54.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 07271293-b24b-44c9-8bee-94358cc9f9fa 2026-03-24T02:41:54.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:41:54.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:54.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_35 snap 2026-03-24T02:41:54.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:54.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:41:54.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:54.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:41:54.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.export 2026-03-24T02:41:54.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_35.export 2026-03-24T02:41:54.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_35.export 2026-03-24T02:41:54.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_35@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.export 2026-03-24T02:41:54.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:54.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_35@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_35.export 2026-03-24T02:41:54.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:54.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_35.export 2026-03-24T02:41:54.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_35.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_35.export 2026-03-24T02:41:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:41:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_36 2026-03-24T02:41:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_36 snap 2026-03-24T02:41:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_36 2026-03-24T02:41:54.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:54.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_36@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:41:55.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_36 2026-03-24T02:41:55.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:55.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_36 2026-03-24T02:41:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_36 started 2026-03-24T02:41:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_36 2026-03-24T02:41:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:41:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:55.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:56.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_36 started 2026-03-24T02:41:56.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:56.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:56.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_36 2026-03-24T02:41:56.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:41:56.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:41:56.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:41:56.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_36 2026-03-24T02:41:56.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:41:56.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:41:56.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:56.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:56.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:41:56.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:41:56.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:56.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:41:56.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:56.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_36 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:41:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:56.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:56.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:56.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:41:56.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:56.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:56.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:56.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:56.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:56.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:56.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:56.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:41:56.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:41:56.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:56.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:56.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:56.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:56.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_36 2026-03-24T02:41:56.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-03-24T02:41:56.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:41:56.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:56.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:56.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_36 2026-03-24T02:41:56.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:41:56.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:41:56.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:56.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:56.110 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:41:56.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:56.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:56.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:56.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:56.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_36 2026-03-24T02:41:56.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:41:56.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:41:56.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:41:56.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:41:56.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:41:56.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:41:56.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:41:56.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:56.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_36 2026-03-24T02:41:56.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:56.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:56.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:56.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:56.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_36 2026-03-24T02:41:56.192 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:41:56.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_36 2026-03-24T02:41:56.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:41:56.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:56.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:41:56.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:41:56.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_36 2026-03-24T02:41:56.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:41:56.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:41:56.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:41:56.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:41:56.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:41:56.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_36 2026-03-24T02:41:56.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:56.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:56.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_36 2026-03-24T02:41:56.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:41:56.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_36 ']' 2026-03-24T02:41:56.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_36' 2026-03-24T02:41:56.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_36 2026-03-24T02:41:56.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:41:56.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:56.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:56.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:41:56.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:41:56.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:56.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:41:56.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:41:56.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:41:56.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:56.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_36 2026-03-24T02:41:56.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:41:56.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:56.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:56.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:41:56.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:41:56.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:56.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:56.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:56.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:41:56.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:56.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:56.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:56.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:41:56.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:56.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:41:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:41:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:41:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:41:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:41:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:41:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:41:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:41:56.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:41:56.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:56.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:56.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:41:56.409 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_36 2026-03-24T02:41:56.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-03-24T02:41:56.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:41:56.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:41:56.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:41:56.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_36 2026-03-24T02:41:56.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:41:56.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:41:56.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:41:56.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:41:56.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:41:56.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:41:56.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:41:56.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:41:56.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:41:56.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:56.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:41:56.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:41:56.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:41:56.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:56.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:56.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:41:56.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_36 2026-03-24T02:41:56.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:56.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_36 2026-03-24T02:41:56.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:56.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:56.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_36 2026-03-24T02:41:56.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_36 '' 2026-03-24T02:41:56.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:56.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:56.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_36 2026-03-24T02:41:56.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:41:56.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-03-24T02:41:56.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_36.status 2026-03-24T02:41:56.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:56.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_36.status 2026-03-24T02:41:56.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-03-24T02:41:56.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_36.status 2026-03-24T02:41:56.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:41:56.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:41:56.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:56.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:56.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:56.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_36.status 2026-03-24T02:41:56.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:56.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_36 2026-03-24T02:41:56.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:56.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:56.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_36 2026-03-24T02:41:56.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_36 '..*' 2026-03-24T02:41:56.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:41:56.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:41:56.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_36 2026-03-24T02:41:56.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:41:56.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-03-24T02:41:56.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_36.status 2026-03-24T02:41:56.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:41:56.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_36.status 2026-03-24T02:41:56.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-03-24T02:41:56.590 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_36.status 2026-03-24T02:41:56.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:41:56.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:41:56.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:41:56.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:56.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:41:56.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_36.status 2026-03-24T02:41:56.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:56.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:41:56.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:56.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_36 snap 2026-03-24T02:41:56.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:56.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:56.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_36 2026-03-24T02:41:56.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:56.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:56.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:41:56.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:41:57.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_36@snap 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_36': 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 12097957c6a5 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.12097957c6a5 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:31:02 2026 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:31:02 2026 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:38:51 2026 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 12097957c6a5 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 50ac70b5-b9cf-4f8c-b5ef-8c362c6d7029 2026-03-24T02:41:57.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:41:57.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:57.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_36 snap 2026-03-24T02:41:57.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:57.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_36 2026-03-24T02:41:57.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:57.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:41:57.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_36.export 2026-03-24T02:41:57.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_36.export 2026-03-24T02:41:57.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_36.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_36.export 2026-03-24T02:41:57.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_36@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_36.export 2026-03-24T02:41:57.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:57.834 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_36@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_36.export 2026-03-24T02:41:58.008 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:41:58.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_36.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_36.export 2026-03-24T02:41:58.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_36.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_36.export 2026-03-24T02:41:58.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:41:58.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:41:58.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_37 2026-03-24T02:41:58.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_37 snap 2026-03-24T02:41:58.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:41:58.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:58.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:41:58.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:41:58.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_37@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:41:59.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_37 2026-03-24T02:41:59.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:59.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:59.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:41:59.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_37 started 2026-03-24T02:41:59.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:41:59.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:41:59.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:41:59.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:41:59.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:41:59.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:41:59.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:00.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_37 started 2026-03-24T02:42:00.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:00.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:00.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:42:00.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:42:00.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:42:00.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:42:00.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_37 2026-03-24T02:42:00.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:42:00.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:42:00.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:00.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:00.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:42:00.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:42:00.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:00.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:42:00.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:00.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:42:00.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:00.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_37 2026-03-24T02:42:00.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:42:00.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:00.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:00.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:00.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:00.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:00.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:00.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:00.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:00.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:00.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:00.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:00.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:00.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:00.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:00.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:42:00.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:00.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:00.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:00.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:42:00.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:00.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:00.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:00.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:00.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:00.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:00.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:00.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:42:00.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:42:00.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:00.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:00.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:00.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:00.104 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_37 2026-03-24T02:42:00.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-03-24T02:42:00.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:42:00.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:00.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:00.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_37 2026-03-24T02:42:00.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:42:00.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:00.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:00.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:00.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:00.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:00.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:00.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:00.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:00.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:00.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:00.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:00.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:00.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:00.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:00.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:42:00.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:00.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:00.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:00.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:42:00.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:00.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:00.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:00.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:00.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:00.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:00.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:00.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:42:00.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:42:00.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:00.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:00.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:00.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:00.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_37 2026-03-24T02:42:00.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:42:00.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:42:00.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:42:00.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_37 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_37 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:00.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:42:00.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_37 2026-03-24T02:42:00.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:00.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:00.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:42:00.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:00.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_37 ']' 2026-03-24T02:42:00.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-03-24T02:42:00.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-03-24T02:42:00.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:00.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:00.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:00.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:00.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:00.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:00.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:00.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:00.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:00.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:00.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-03-24T02:42:00.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:00.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:00.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:00.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:00.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:00.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:00.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:00.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:00.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:00.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:00.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:00.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:00.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:00.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:00.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:00.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:00.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:00.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:00.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-03-24T02:42:00.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-03-24T02:42:00.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:00.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:00.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:00.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-03-24T02:42:00.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:00.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:00.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:00.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:00.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:00.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:00.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:00.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:00.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:00.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:00.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:00.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:00.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:00.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:00.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:00.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-03-24T02:42:00.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:00.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-03-24T02:42:00.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:00.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:00.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:00.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-03-24T02:42:00.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:00.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:00.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:00.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:00.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-03-24T02:42:00.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_37.status 2026-03-24T02:42:00.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:00.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:00.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-03-24T02:42:00.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:00.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:00.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:00.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:00.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:00.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:00.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-03-24T02:42:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-03-24T02:42:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:00.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-03-24T02:42:00.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_37.status 2026-03-24T02:42:00.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:00.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:00.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-03-24T02:42:00.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:00.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:00.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:00.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:00.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:00.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:00.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:00.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:00.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:00.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:42:00.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:42:00.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:42:00.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:00.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:00.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:42:00.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:00.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:42:00.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:42:00.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:00.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:42:00.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:00.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:42:00.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:00.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:42:00.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:00.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:42:00.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:00.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:42:00.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:42:00.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:42:00.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:00.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:42:00.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_37 2026-03-24T02:42:00.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:00.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:00.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:42:00.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:00.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_37 ']' 2026-03-24T02:42:00.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-03-24T02:42:00.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-03-24T02:42:00.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:00.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:00.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:00.878 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:00.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:00.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:00.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:00.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:00.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:00.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:00.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-03-24T02:42:00.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:00.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:00.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:00.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:00.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:00.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:00.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:00.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:00.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:00.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:00.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:00.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:00.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:00.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:00.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:00.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:00.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:00.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:00.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:00.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:00.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:00.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:00.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:00.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:00.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:00.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:00.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:00.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:00.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:00.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:00.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:00.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:00.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:00.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-03-24T02:42:00.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-03-24T02:42:00.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:00.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:00.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:00.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-03-24T02:42:00.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:00.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:00.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:00.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:00.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:00.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:00.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:00.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:00.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:00.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:00.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:00.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:00.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:00.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:00.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:00.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:00.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:00.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:00.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:00.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:00.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:00.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:00.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:00.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:00.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:00.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:00.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:00.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:00.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:00.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:00.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:00.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:00.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:00.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-03-24T02:42:01.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:01.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-03-24T02:42:01.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:01.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:01.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:01.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-03-24T02:42:01.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:01.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:01.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:01.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:01.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-03-24T02:42:01.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_37.status 2026-03-24T02:42:01.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:01.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:01.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-03-24T02:42:01.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:01.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:01.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:01.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:01.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:01.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:01.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:01.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:01.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-03-24T02:42:01.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:01.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:01.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:01.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-03-24T02:42:01.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:01.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:01.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:01.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:01.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-03-24T02:42:01.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_37.status 2026-03-24T02:42:01.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:01.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:01.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-03-24T02:42:01.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:01.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:01.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:01.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:01.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:01.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:01.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:01.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:01.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:01.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:01.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:01.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:42:01.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_37 2026-03-24T02:42:01.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:01.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:01.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:42:01.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:01.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_37 ']' 2026-03-24T02:42:01.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-03-24T02:42:01.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-03-24T02:42:01.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:01.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:01.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:01.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:01.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:01.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:01.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:01.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:01.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:01.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:01.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:01.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:01.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:01.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:01.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:01.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-03-24T02:42:01.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-03-24T02:42:01.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:01.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:01.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:01.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:01.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:01.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:01.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-03-24T02:42:01.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:01.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-03-24T02:42:01.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:01.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:01.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:01.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-03-24T02:42:01.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:01.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:01.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:01.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:01.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-03-24T02:42:01.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_37.status 2026-03-24T02:42:01.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:01.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:01.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-03-24T02:42:01.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:01.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:01.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:01.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:01.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:01.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:01.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:01.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-03-24T02:42:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-03-24T02:42:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-03-24T02:42:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_37.status 2026-03-24T02:42:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:01.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:01.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-03-24T02:42:01.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:01.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:01.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:01.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:01.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:01.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:01.707 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:01.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:01.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:01.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:01.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:01.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:42:02.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_37 2026-03-24T02:42:02.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:02.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:02.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:42:02.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:02.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_37 ']' 2026-03-24T02:42:02.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-03-24T02:42:02.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-03-24T02:42:02.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:02.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:02.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:02.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:02.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:02.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:02.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:02.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:02.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-03-24T02:42:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:02.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:02.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:02.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:02.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:02.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:02.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-03-24T02:42:02.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-03-24T02:42:02.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:02.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:02.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:02.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:02.599 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-03-24T02:42:02.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:02.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-03-24T02:42:02.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:02.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:02.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:02.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-03-24T02:42:02.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:02.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:02.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:02.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:02.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-03-24T02:42:02.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_37.status 2026-03-24T02:42:02.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:02.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:02.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-03-24T02:42:02.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:02.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:02.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:02.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:02.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:02.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:02.693 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:02.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:02.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-03-24T02:42:02.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:02.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:02.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:02.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-03-24T02:42:02.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:02.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:02.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:02.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:02.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-03-24T02:42:02.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_37.status 2026-03-24T02:42:02.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:02.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:02.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-03-24T02:42:02.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:02.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:02.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:02.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:02.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:02.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:02.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:02.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:02.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:02.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:02.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:02.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:42:04.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_37 2026-03-24T02:42:04.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:04.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:04.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:42:04.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:04.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_37 ']' 2026-03-24T02:42:04.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-03-24T02:42:04.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-03-24T02:42:04.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:04.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:04.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:04.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:04.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:04.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:04.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:04.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:04.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:04.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:04.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-03-24T02:42:04.330 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:04.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:04.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:04.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:04.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:04.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-03-24T02:42:04.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-03-24T02:42:04.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:04.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:04.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:04.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-03-24T02:42:04.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:04.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:04.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:04.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:04.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:04.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:04.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:04.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:04.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:04.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:04.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:04.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:04.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:04.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-03-24T02:42:04.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:04.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-03-24T02:42:04.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:04.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:04.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:04.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-03-24T02:42:04.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:04.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:04.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:04.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:04.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-03-24T02:42:04.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_37.status 2026-03-24T02:42:04.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:04.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:04.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-03-24T02:42:04.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:04.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:04.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:04.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:04.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:04.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:04.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:04.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:04.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-03-24T02:42:04.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:04.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:04.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:04.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-03-24T02:42:04.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:04.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:04.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:42:04.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:04.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-03-24T02:42:04.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_37.status 2026-03-24T02:42:04.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:04.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:04.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-03-24T02:42:04.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:04.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:04.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:04.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:04.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:04.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:04.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.status 2026-03-24T02:42:04.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:04.540 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:04.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:04.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_37 snap 2026-03-24T02:42:04.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:04.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:04.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:42:04.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:04.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:42:04.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:42:04.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:05.542 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_37@snap 2026-03-24T02:42:05.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_37': 2026-03-24T02:42:05.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:42:05.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:42:05.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:42:05.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c911b17aa1 2026-03-24T02:42:05.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c911b17aa1 2026-03-24T02:42:05.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:42:05.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:42:05.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:42:05.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:42:05.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:31:14 2026 2026-03-24T02:42:05.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:31:14 2026 2026-03-24T02:42:05.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:37:22 2026 2026-03-24T02:42:05.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:42:05.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c911b17aa1 2026-03-24T02:42:05.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:42:05.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:42:05.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: bcb47fec-197c-4732-aadf-1c79c8f5966e 2026-03-24T02:42:05.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:42:05.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:05.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_37 snap 2026-03-24T02:42:05.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:05.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:42:05.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:05.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:42:05.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.export 2026-03-24T02:42:05.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_37.export 2026-03-24T02:42:05.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_37.export 2026-03-24T02:42:05.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_37@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.export 2026-03-24T02:42:05.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:05.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_37@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_37.export 2026-03-24T02:42:05.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:05.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_37.export 2026-03-24T02:42:05.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_37.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_37.export 2026-03-24T02:42:06.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:06.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:42:06.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_38 2026-03-24T02:42:06.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_38 snap 2026-03-24T02:42:06.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:06.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:06.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:42:06.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:06.006 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_38@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:42:07.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_38 2026-03-24T02:42:07.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:07.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:07.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:42:07.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_38 started 2026-03-24T02:42:07.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:07.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:07.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:42:07.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:42:07.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:42:07.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:07.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:08.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_38 started 2026-03-24T02:42:08.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:08.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:08.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:42:08.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:42:08.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:42:08.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:42:08.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_38 2026-03-24T02:42:08.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:42:08.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:42:08.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:08.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:08.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:42:08.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:42:08.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:08.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:42:08.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:08.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:42:08.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_38 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:08.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:08.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:08.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:08.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:08.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:08.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:42:08.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:42:08.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:08.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:08.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:08.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:08.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_38 2026-03-24T02:42:08.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:42:08.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:42:08.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:42:08.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:42:08.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_38 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_38 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:08.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:42:08.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_38 2026-03-24T02:42:08.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:08.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:08.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:42:08.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:08.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_38 ']' 2026-03-24T02:42:08.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-03-24T02:42:08.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-03-24T02:42:08.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:08.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:08.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:08.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:08.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:08.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:08.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:08.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:08.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:08.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:08.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:08.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:08.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:08.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:08.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:08.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:08.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:08.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:08.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:08.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:08.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:08.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:08.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:08.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:08.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-03-24T02:42:08.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:08.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-03-24T02:42:08.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:08.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:08.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:08.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-03-24T02:42:08.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:08.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:08.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:08.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:08.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-03-24T02:42:08.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_38.status 2026-03-24T02:42:08.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:08.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:08.575 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-03-24T02:42:08.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:08.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:08.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:08.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:08.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:08.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:08.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:08.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:08.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-03-24T02:42:08.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:08.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:08.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:08.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-03-24T02:42:08.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:08.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:08.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:08.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:08.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-03-24T02:42:08.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_38.status 2026-03-24T02:42:08.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:08.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:08.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-03-24T02:42:08.609 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:08.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:08.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:08.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:08.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:08.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:08.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:08.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:08.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:08.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:42:08.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:42:08.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:42:08.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:08.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:08.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:42:08.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:08.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:42:08.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:08.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:42:08.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:08.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:42:08.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:08.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:42:08.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:08.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:42:08.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:08.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:42:08.645 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:08.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:42:08.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:42:08.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:42:08.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:08.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:42:08.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_38 2026-03-24T02:42:08.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:08.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:08.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:42:08.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:08.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_38 ']' 2026-03-24T02:42:08.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-03-24T02:42:08.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-03-24T02:42:08.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:08.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:08.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:08.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:08.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:08.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:08.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:08.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:08.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-03-24T02:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:08.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:08.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:08.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:08.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:08.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:08.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:08.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:08.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:08.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:08.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:08.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:08.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:08.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:08.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:08.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:08.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:08.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:08.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:08.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-03-24T02:42:08.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-03-24T02:42:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-03-24T02:42:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-03-24T02:42:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_38.status 2026-03-24T02:42:08.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:08.927 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:08.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-03-24T02:42:08.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:08.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:08.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:08.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:08.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:08.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:08.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:08.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:08.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-03-24T02:42:08.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:08.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:08.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:08.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-03-24T02:42:08.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:08.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:08.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:08.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:08.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-03-24T02:42:08.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_38.status 2026-03-24T02:42:08.960 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:08.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:08.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-03-24T02:42:08.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:08.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:08.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:08.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:08.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:08.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:08.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:08.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:08.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:08.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:08.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:08.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:42:09.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_38 2026-03-24T02:42:09.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:09.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:09.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:42:09.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:09.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_38 ']' 2026-03-24T02:42:09.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-03-24T02:42:09.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-03-24T02:42:09.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:09.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:09.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:09.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:09.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:09.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:09.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:09.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:09.397 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-03-24T02:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:09.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:09.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:09.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:09.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-03-24T02:42:09.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:09.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-03-24T02:42:09.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:09.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:09.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:09.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-03-24T02:42:09.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:09.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:09.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:09.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:09.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-03-24T02:42:09.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_38.status 2026-03-24T02:42:09.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:09.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:09.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-03-24T02:42:09.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:09.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:09.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:09.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:09.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:09.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:09.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:09.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:09.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-03-24T02:42:09.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:09.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:09.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:09.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-03-24T02:42:09.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:09.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:09.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:09.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:09.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-03-24T02:42:09.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_38.status 2026-03-24T02:42:09.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:09.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:09.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-03-24T02:42:09.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:09.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:09.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:09.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:09.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:09.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:09.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:09.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:09.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:09.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:09.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:09.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:42:10.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_38 2026-03-24T02:42:10.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:10.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:10.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:42:10.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:10.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_38 ']' 2026-03-24T02:42:10.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-03-24T02:42:10.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-03-24T02:42:10.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:10.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:10.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:10.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:10.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:10.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:10.349 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:10.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:10.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-03-24T02:42:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:10.351 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:10.352 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:10.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:10.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:10.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:10.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-03-24T02:42:10.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:10.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-03-24T02:42:10.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:10.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:10.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:10.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-03-24T02:42:10.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:10.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:10.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:10.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:10.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-03-24T02:42:10.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_38.status 2026-03-24T02:42:10.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:10.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:10.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-03-24T02:42:10.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:10.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:10.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:10.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:10.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:10.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:10.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:10.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:10.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-03-24T02:42:10.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:10.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:10.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:10.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-03-24T02:42:10.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:10.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:10.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:10.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:10.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-03-24T02:42:10.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_38.status 2026-03-24T02:42:10.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:10.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:10.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-03-24T02:42:10.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:10.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:10.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:10.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:10.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:10.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:10.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:10.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_38 2026-03-24T02:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_38 ']' 2026-03-24T02:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-03-24T02:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-03-24T02:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:12.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:12.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:12.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:12.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:12.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:12.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:12.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:12.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-03-24T02:42:12.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:12.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:12.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:12.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:12.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:12.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:12.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:12.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:12.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:12.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:12.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-03-24T02:42:12.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:12.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-03-24T02:42:12.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:12.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:12.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:12.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-03-24T02:42:12.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:12.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:12.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:12.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:12.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-03-24T02:42:12.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_38.status 2026-03-24T02:42:12.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:12.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:12.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-03-24T02:42:12.191 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:12.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:12.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:12.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:12.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:12.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:12.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:12.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:12.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-03-24T02:42:12.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:12.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:12.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:12.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-03-24T02:42:12.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:12.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:12.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:42:12.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:12.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-03-24T02:42:12.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_38.status 2026-03-24T02:42:12.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:12.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:12.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-03-24T02:42:12.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:12.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:12.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:12.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:12.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:12.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:12.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.status 2026-03-24T02:42:12.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:12.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:12.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:12.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_38 snap 2026-03-24T02:42:12.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:12.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:12.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:42:12.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:12.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:42:12.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:42:12.265 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:13.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_38@snap 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_38': 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10d118d230f8 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10d118d230f8 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:31:23 2026 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:31:23 2026 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:38:54 2026 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10d118d230f8 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 05004e8b-72b7-4b81-98a0-5ca5620caf74 2026-03-24T02:42:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:42:13.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:13.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_38 snap 2026-03-24T02:42:13.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:13.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:42:13.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:13.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:42:13.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.export 2026-03-24T02:42:13.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_38.export 2026-03-24T02:42:13.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_38.export 2026-03-24T02:42:13.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_38@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.export 2026-03-24T02:42:13.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:13.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_38@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_38.export 2026-03-24T02:42:13.668 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:13.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_38.export 2026-03-24T02:42:13.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_38.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_38.export 2026-03-24T02:42:13.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:13.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:42:13.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_39 2026-03-24T02:42:13.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_39 snap 2026-03-24T02:42:13.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:13.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:13.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:42:13.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:13.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_39@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:42:14.214 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8695463106294532) > weight (0.25) 2026-03-24T02:42:14.214 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.761623017531878) > weight (0.25) 2026-03-24T02:42:14.214 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.640117746492107) > weight (0.25) 2026-03-24T02:42:14.214 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-03-24T02:42:14.214 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-03-24T02:42:14.214 INFO:tasks.rbd_mirror_thrash:waiting for 7.573973660802205 secs before reviving daemons 2026-03-24T02:42:14.214 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:42:14.209+0000 7f414b311640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 26423) UID: 1000 2026-03-24T02:42:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_39 2026-03-24T02:42:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:42:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_39 started 2026-03-24T02:42:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:42:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:42:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:42:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:15.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_39 started 2026-03-24T02:42:15.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:15.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:15.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:42:15.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:42:15.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:42:15.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:42:15.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_39 2026-03-24T02:42:15.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:42:15.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:15.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:42:15.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:15.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:42:15.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:42:15.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:15.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:42:15.254 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:15.255 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_39 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:15.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:15.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:15.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:42:15.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:42:15.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:15.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:15.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:15.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:15.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_39 2026-03-24T02:42:15.333 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:42:15.325+0000 7ff7e5b71640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55e9e68f8380 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-03-24T02:42:15.333 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:42:15.325+0000 7ff7e5b71640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55e9e68f8380 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-03-24T02:42:15.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-03-24T02:42:15.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:42:15.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:15.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:15.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_39 2026-03-24T02:42:15.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:42:15.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:15.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:15.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:15.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:15.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:15.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:15.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:15.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:15.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:15.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:15.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:15.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:15.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:15.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:15.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:42:15.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:15.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:15.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:15.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:42:15.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:15.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:15.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:15.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:15.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:15.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:15.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:15.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:42:15.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:42:15.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:15.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:15.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:15.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:15.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_39 2026-03-24T02:42:15.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-03-24T02:42:15.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:42:15.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:15.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:15.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_39 2026-03-24T02:42:15.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:42:15.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:15.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:15.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:15.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:15.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:15.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:15.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:15.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:15.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:15.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:15.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:15.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:15.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:15.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:15.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:42:15.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:15.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:15.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:15.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:42:15.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:15.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:15.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:15.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:15.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:15.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:15.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:15.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:42:15.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:42:15.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:15.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:15.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:15.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:15.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_39 2026-03-24T02:42:15.780 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:42:15.774+0000 7f5496ce4640 -1 reset not still connected to 0x5652d714e340 2026-03-24T02:42:15.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:42:15.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:42:15.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:42:15.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:42:15.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:42:15.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:42:15.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:42:15.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:15.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_39 2026-03-24T02:42:15.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:15.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:15.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:15.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:15.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:42:15.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:42:15.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_39 2026-03-24T02:42:15.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:15.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:15.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:15.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:15.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:42:15.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:42:15.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:42:15.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:42:15.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:15.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:42:15.929 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:42:15.921+0000 7f085e5d7640 -1 reset not still connected to 0x5587a1fe9c70 2026-03-24T02:42:15.929 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:42:15.921+0000 7f085e5d7640 -1 reset not still connected to 0x5587a7cbe270 2026-03-24T02:42:16.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_39 2026-03-24T02:42:16.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:16.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:16.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:42:16.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:16.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_39 ']' 2026-03-24T02:42:16.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-03-24T02:42:16.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-03-24T02:42:16.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:16.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:16.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:16.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:16.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:16.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:16.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:16.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:16.026 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:16.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:16.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-03-24T02:42:16.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:16.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:16.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:16.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:16.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:16.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:16.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:16.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:16.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:16.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:16.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:16.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:16.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:16.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:16.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:16.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:16.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:16.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:16.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:16.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:16.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:16.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:16.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:16.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:16.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:16.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:16.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:16.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:16.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:16.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:16.078 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:16.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:16.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:16.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-03-24T02:42:16.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-03-24T02:42:16.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:16.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-03-24T02:42:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:16.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:16.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:16.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:16.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:16.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:16.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:16.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-03-24T02:42:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-03-24T02:42:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_39 2026-03-24T02:42:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:42:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:42:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:42:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:42:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:42:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:16.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:16.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:16.475 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:16.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_39 2026-03-24T02:42:17.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:17.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-03-24T02:42:17.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:17.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:17.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_39 2026-03-24T02:42:17.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-03-24T02:42:17.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:17.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:17.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_39 2026-03-24T02:42:17.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:17.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-03-24T02:42:17.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_39.status 2026-03-24T02:42:17.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:17.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_39.status 2026-03-24T02:42:17.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-03-24T02:42:17.041 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_39.status 2026-03-24T02:42:17.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:17.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:17.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:17.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:17.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:17.082 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_39.status 2026-03-24T02:42:17.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:17.083 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-03-24T02:42:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_39 2026-03-24T02:42:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-03-24T02:42:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_39 2026-03-24T02:42:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-03-24T02:42:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_39.status 2026-03-24T02:42:17.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:17.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_39.status 2026-03-24T02:42:17.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-03-24T02:42:17.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_39.status 2026-03-24T02:42:17.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:17.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:17.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:17.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:17.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:17.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_39.status 2026-03-24T02:42:17.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:17.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:17.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:42:17.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:42:17.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:42:17.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:17.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:17.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:42:17.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:17.127 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:42:17.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:17.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:17.128 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:42:17.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:42:17.129 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:17.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:42:17.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:17.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:42:17.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:17.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:42:17.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:17.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:42:17.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:42:17.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:42:17.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:17.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_39 2026-03-24T02:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_39 ']' 2026-03-24T02:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-03-24T02:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-03-24T02:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:17.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:17.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-03-24T02:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:17.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:17.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:17.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:17.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-03-24T02:42:17.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-03-24T02:42:17.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:17.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:17.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:17.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:17.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:17.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-03-24T02:42:17.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-03-24T02:42:17.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:17.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:17.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:17.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_39 2026-03-24T02:42:17.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:42:17.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:17.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:17.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:17.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:42:17.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:17.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:17.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:17.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:42:17.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:17.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:17.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:17.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:42:17.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:17.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:17.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:42:17.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:17.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:17.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:17.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:17.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:17.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:17.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:17.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:17.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:17.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:17.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:17.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:17.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:17.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:17.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:17.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:17.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:17.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_39 2026-03-24T02:42:17.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:17.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-03-24T02:42:17.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:17.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:17.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_39 2026-03-24T02:42:17.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-03-24T02:42:17.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:17.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:17.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_39 2026-03-24T02:42:17.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:17.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-03-24T02:42:17.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_39.status 2026-03-24T02:42:17.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:17.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_39.status 2026-03-24T02:42:17.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-03-24T02:42:17.606 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_39.status 2026-03-24T02:42:17.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:17.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:17.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:17.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:17.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:17.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_39.status 2026-03-24T02:42:17.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:17.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-03-24T02:42:17.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:17.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:17.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_39 2026-03-24T02:42:17.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-03-24T02:42:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_39 2026-03-24T02:42:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-03-24T02:42:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_39.status 2026-03-24T02:42:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:17.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_39.status 2026-03-24T02:42:17.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-03-24T02:42:17.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_39.status 2026-03-24T02:42:17.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:17.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:17.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:17.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:17.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:17.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_39.status 2026-03-24T02:42:17.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:17.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:17.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:17.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_39 snap 2026-03-24T02:42:17.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:17.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:17.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:42:17.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:17.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:42:17.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:42:17.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:18.358 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:42:18.349+0000 7f085e5d7640 -1 reset not still connected to 0x5587a59896c0 2026-03-24T02:42:18.358 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:42:18.349+0000 7f085e5d7640 -1 reset not still connected to 0x5587bad53e10 2026-03-24T02:42:18.425 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:42:18.418+0000 7f5496ce4640 -1 reset not still connected to 0x5652c92ea5b0 2026-03-24T02:42:18.427 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:42:18.422+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c30d00 2026-03-24T02:42:18.428 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:42:18.422+0000 7f5496ce4640 -1 reset not still connected to 0x5652d1c5d110 2026-03-24T02:42:18.428 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:42:18.422+0000 7f5496ce4640 -1 reset not still connected to 0x5652d714f930 2026-03-24T02:42:18.428 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:42:18.422+0000 7f5496ce4640 -1 reset not still connected to 0x5652ef236ea0 2026-03-24T02:42:18.689 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_39@snap 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_39': 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 116de22d20ce 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.116de22d20ce 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:31:38 2026 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:31:38 2026 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:38:49 2026 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 116de22d20ce 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 3e3da622-c260-4bd0-9c8e-c88fefda0de4 2026-03-24T02:42:18.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:42:18.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:18.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_39 snap 2026-03-24T02:42:18.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:18.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:42:18.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:18.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:42:18.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_39.export 2026-03-24T02:42:18.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_39.export 2026-03-24T02:42:18.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_39.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_39.export 2026-03-24T02:42:18.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_39@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_39.export 2026-03-24T02:42:18.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:18.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_39@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_39.export 2026-03-24T02:42:19.197 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:19.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_39.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_39.export 2026-03-24T02:42:19.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_39.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_39.export 2026-03-24T02:42:19.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:19.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:42:19.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_40 2026-03-24T02:42:19.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_40 snap 2026-03-24T02:42:19.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:19.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:19.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:42:19.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:19.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_40@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:42:20.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_40 2026-03-24T02:42:20.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:20.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:20.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:42:20.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_40 started 2026-03-24T02:42:20.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:20.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:20.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:42:20.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:42:20.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:42:20.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:20.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:21.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_40 started 2026-03-24T02:42:21.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:21.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:21.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:42:21.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:42:21.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:42:21.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:42:21.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_40 2026-03-24T02:42:21.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:42:21.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:42:21.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:21.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:21.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:42:21.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:42:21.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:21.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:42:21.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:21.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:42:21.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:21.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_40 2026-03-24T02:42:21.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:42:21.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:21.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:21.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:21.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:21.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:21.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:21.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:21.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:21.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:21.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:21.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:21.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:21.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:21.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:21.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:42:21.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:21.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:21.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:21.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:42:21.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:21.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:21.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:21.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:21.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:21.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:21.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:21.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:42:21.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:42:21.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:21.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:21.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:21.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:21.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_40 2026-03-24T02:42:21.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-03-24T02:42:21.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:42:21.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:21.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:21.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_40 2026-03-24T02:42:21.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:42:21.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:21.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:21.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:21.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:42:21.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:42:21.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:21.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:21.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:21.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:21.474 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_40 2026-03-24T02:42:21.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:42:21.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:42:21.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:42:21.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:42:21.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:42:21.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:42:21.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:42:21.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:21.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_40 2026-03-24T02:42:21.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:21.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:21.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:21.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:21.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:42:21.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:42:21.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_40 2026-03-24T02:42:21.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:21.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:21.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:21.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:21.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:42:21.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:42:21.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:42:21.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:42:21.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:21.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:42:21.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_40 2026-03-24T02:42:21.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:21.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:21.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:42:21.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:21.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_40 ']' 2026-03-24T02:42:21.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-03-24T02:42:21.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-03-24T02:42:21.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:21.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:21.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:21.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:21.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:21.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:21.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:21.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:21.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:21.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:21.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-03-24T02:42:21.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:21.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:21.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:21.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:21.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:21.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:21.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:21.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:21.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:21.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:21.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:21.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:21.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:21.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:21.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:21.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:21.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:21.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:21.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:21.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:21.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:21.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:21.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:21.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:21.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:21.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:21.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:21.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:21.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:21.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:21.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:21.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:21.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:21.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-03-24T02:42:21.789 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-03-24T02:42:21.789 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:42:21.789 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-03-24T02:42:21.789 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-03-24T02:42:21.789 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-03-24T02:42:21.790 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-03-24T02:42:21.790 INFO:tasks.rbd_mirror_thrash:waiting for 76.88197260868952 secs before thrashing 2026-03-24T02:42:21.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-03-24T02:42:21.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:21.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2026-03-24T02:42:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:21.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:21.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:21.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:21.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:21.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:21.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2026-03-24T02:42:21.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:21.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-03-24T02:42:21.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:21.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:21.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_40 2026-03-24T02:42:21.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-03-24T02:42:21.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:21.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:21.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_40 2026-03-24T02:42:21.956 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:21.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-03-24T02:42:21.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_40.status 2026-03-24T02:42:21.957 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:21.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_40.status 2026-03-24T02:42:21.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_40.status 2026-03-24T02:42:21.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-03-24T02:42:22.002 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:22.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:22.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:22.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:22.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:22.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_40.status 2026-03-24T02:42:22.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:22.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-03-24T02:42:22.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:22.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:22.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_40 2026-03-24T02:42:22.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-03-24T02:42:22.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:22.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:22.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_40 2026-03-24T02:42:22.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:22.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-03-24T02:42:22.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_40.status 2026-03-24T02:42:22.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:22.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_40.status 2026-03-24T02:42:22.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-03-24T02:42:22.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_40.status 2026-03-24T02:42:22.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:22.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:22.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:22.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:22.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:22.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_40.status 2026-03-24T02:42:22.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:22.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:22.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:42:22.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:42:22.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:42:22.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:22.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:22.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:22.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:42:22.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:42:22.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:22.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:42:22.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:22.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:42:22.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:22.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:42:22.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:22.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:42:22.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:22.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:42:22.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:22.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:42:22.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:42:22.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:42:22.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:22.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:42:22.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_40 2026-03-24T02:42:22.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:22.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:22.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:42:22.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:22.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_40 ']' 2026-03-24T02:42:22.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-03-24T02:42:22.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-03-24T02:42:22.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:22.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:22.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:22.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:22.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:22.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:22.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:22.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:22.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:22.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:22.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-03-24T02:42:22.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:22.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:22.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:22.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:22.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:22.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:22.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:22.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:22.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:22.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:22.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:22.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:22.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:22.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:22.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:22.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:22.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:22.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:22.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:22.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:22.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:22.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:22.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:22.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:22.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:22.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:22.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:22.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:22.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:22.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:22.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-03-24T02:42:22.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-03-24T02:42:22.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:22.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:22.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:22.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2026-03-24T02:42:22.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:22.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:22.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:22.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:22.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:22.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:22.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:22.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:22.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:22.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:22.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:22.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:22.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:22.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:22.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:22.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2026-03-24T02:42:22.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:22.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-03-24T02:42:22.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:22.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:22.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_40 2026-03-24T02:42:22.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-03-24T02:42:22.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:22.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:22.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_40 2026-03-24T02:42:22.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:22.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-03-24T02:42:22.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_40.status 2026-03-24T02:42:22.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:22.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_40.status 2026-03-24T02:42:22.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-03-24T02:42:22.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_40.status 2026-03-24T02:42:22.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:22.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:22.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:22.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:22.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:22.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_40.status 2026-03-24T02:42:22.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:22.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-03-24T02:42:22.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:22.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:22.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_40 2026-03-24T02:42:22.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-03-24T02:42:22.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:22.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:22.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_40 2026-03-24T02:42:22.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:22.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-03-24T02:42:22.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_40.status 2026-03-24T02:42:22.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:22.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_40.status 2026-03-24T02:42:22.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-03-24T02:42:22.491 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_40.status 2026-03-24T02:42:22.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:22.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:22.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:22.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:22.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:22.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_40.status 2026-03-24T02:42:22.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:22.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:22.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:22.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_40 snap 2026-03-24T02:42:22.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:22.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:22.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:42:22.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:22.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:42:22.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:42:22.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:23.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_40@snap 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_40': 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 120916dafe24 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.120916dafe24 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:31:46 2026 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:31:46 2026 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:37:49 2026 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 120916dafe24 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: f5520cc8-7523-4274-b4e6-7db152dd790f 2026-03-24T02:42:23.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:42:23.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:23.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_40 snap 2026-03-24T02:42:23.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:23.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:42:23.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:23.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:42:23.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_40.export 2026-03-24T02:42:23.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_40.export 2026-03-24T02:42:23.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_40.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_40.export 2026-03-24T02:42:23.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_40@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_40.export 2026-03-24T02:42:23.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:23.746 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_40@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_40.export 2026-03-24T02:42:23.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:23.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_40.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_40.export 2026-03-24T02:42:23.987 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_40.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_40.export 2026-03-24T02:42:24.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:24.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:42:24.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_41 2026-03-24T02:42:24.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_41 snap 2026-03-24T02:42:24.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:24.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:24.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:42:24.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:24.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_41@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:42:24.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_41 2026-03-24T02:42:24.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:24.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:24.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:42:24.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_41 started 2026-03-24T02:42:24.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:24.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:24.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:42:24.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:42:24.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:42:24.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:24.378 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:25.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_41 started 2026-03-24T02:42:25.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:25.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:25.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:42:25.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:42:25.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:42:25.379 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:42:25.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_41 2026-03-24T02:42:25.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:42:25.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:42:25.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:25.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:25.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:42:25.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:42:25.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:25.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:42:25.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:25.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:42:25.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:25.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_41 2026-03-24T02:42:25.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:42:25.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:25.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:25.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:25.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:25.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:25.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:25.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:25.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:25.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:42:25.383 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:25.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:25.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:25.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:25.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_41 2026-03-24T02:42:25.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-03-24T02:42:25.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:42:25.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:25.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:25.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_41 2026-03-24T02:42:25.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:42:25.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:42:25.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:25.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:25.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:25.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:25.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_41 2026-03-24T02:42:25.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-03-24T02:42:25.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:42:25.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:25.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:25.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_41 2026-03-24T02:42:25.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:42:25.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:25.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:25.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:25.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:25.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:25.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:25.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:25.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:25.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:42:25.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:42:25.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:25.580 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:25.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:25.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:25.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_41 2026-03-24T02:42:25.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:42:25.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:42:25.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:42:25.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_41 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_41 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:42:25.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:25.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:42:25.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_41 2026-03-24T02:42:25.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:25.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:25.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:42:25.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:25.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_41 ']' 2026-03-24T02:42:25.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_41' 2026-03-24T02:42:25.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2026-03-24T02:42:25.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:25.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:25.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:25.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:25.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:25.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:25.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:25.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:25.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:25.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:25.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2026-03-24T02:42:25.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:25.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:25.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:25.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:25.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:25.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:25.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:25.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:25.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:25.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2026-03-24T02:42:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-03-24T02:42:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:25.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:25.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:25.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:25.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:25.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:25.951 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:25.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:25.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:25.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:25.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2026-03-24T02:42:26.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-03-24T02:42:26.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:26.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_41 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:26.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:42:26.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:26.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:26.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:26.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:26.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:26.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:26.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:26.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:26.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:26.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:26.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:26.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:26.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:26.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:26.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:26.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:26.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:26.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_41 2026-03-24T02:42:26.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:26.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_41 2026-03-24T02:42:26.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:26.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:26.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_41 2026-03-24T02:42:26.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_41 '' 2026-03-24T02:42:26.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:26.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:26.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_41 2026-03-24T02:42:26.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:26.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-03-24T02:42:26.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_41.status 2026-03-24T02:42:26.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:26.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:26.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-03-24T02:42:26.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:26.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:26.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:26.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:26.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:26.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:26.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:26.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:26.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2026-03-24T02:42:26.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:26.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:26.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_41 2026-03-24T02:42:26.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2026-03-24T02:42:26.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:26.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:26.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_41 2026-03-24T02:42:26.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:26.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-03-24T02:42:26.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_41.status 2026-03-24T02:42:26.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:26.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:26.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-03-24T02:42:26.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:26.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:26.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:26.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:26.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:26.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:26.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:26.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:26.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:26.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:42:26.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:42:26.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:42:26.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:26.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:26.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:42:26.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:26.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:42:26.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:26.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:26.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:42:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:42:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:42:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:26.215 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:42:26.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:26.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:42:26.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:26.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:42:26.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:42:26.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:42:26.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:26.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:42:26.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_41 2026-03-24T02:42:26.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:26.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:26.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:42:26.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:26.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_41 ']' 2026-03-24T02:42:26.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_41' 2026-03-24T02:42:26.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2026-03-24T02:42:26.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:26.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:26.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:26.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:26.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:26.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:26.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:26.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:26.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2026-03-24T02:42:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:26.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:26.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:26.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:26.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2026-03-24T02:42:26.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-03-24T02:42:26.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:26.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:26.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:26.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2026-03-24T02:42:26.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:26.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:26.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:26.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:26.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:26.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:26.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:26.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:26.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:26.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:26.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:26.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:26.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:26.506 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:26.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:26.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:26.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:26.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:26.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:26.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:26.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:26.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:26.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:26.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:26.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:26.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:26.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:26.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:26.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:26.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:26.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:26.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:26.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:26.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2026-03-24T02:42:26.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-03-24T02:42:26.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:26.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_41 2026-03-24T02:42:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:42:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:42:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:42:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:42:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:42:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:26.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:26.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:26.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:26.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_41 2026-03-24T02:42:26.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:26.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_41 2026-03-24T02:42:26.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:26.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:26.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_41 2026-03-24T02:42:26.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_41 '' 2026-03-24T02:42:26.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:26.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:26.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_41 2026-03-24T02:42:26.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:26.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-03-24T02:42:26.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_41.status 2026-03-24T02:42:26.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:26.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:26.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-03-24T02:42:26.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:26.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:26.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:26.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:26.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:26.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:26.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:26.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2026-03-24T02:42:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_41 2026-03-24T02:42:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2026-03-24T02:42:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_41 2026-03-24T02:42:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-03-24T02:42:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_41.status 2026-03-24T02:42:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:26.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:26.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-03-24T02:42:26.724 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:26.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:26.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:26.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:26.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:26.757 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:26.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:26.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:26.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:26.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:42:26.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:26.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:42:27.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_41 2026-03-24T02:42:27.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:27.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:27.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:42:27.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:27.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_41 ']' 2026-03-24T02:42:27.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_41' 2026-03-24T02:42:27.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2026-03-24T02:42:27.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:27.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:27.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:27.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:27.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:27.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:27.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:27.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:27.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:27.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:27.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:27.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:27.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:27.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2026-03-24T02:42:27.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-03-24T02:42:27.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:27.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:27.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:27.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2026-03-24T02:42:27.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:27.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:27.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:27.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:27.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:27.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:27.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:27.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:27.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:27.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:27.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:27.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:27.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:27.250 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:27.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:27.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:27.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:27.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:27.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:27.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:27.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:27.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:27.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:27.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:27.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:27.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:27.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:27.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:27.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:27.251 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:27.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:27.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:27.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:27.266 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2026-03-24T02:42:27.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-03-24T02:42:27.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:27.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:27.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:27.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_41 2026-03-24T02:42:27.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:42:27.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:27.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:27.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:27.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:42:27.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:27.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:27.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:27.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:27.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:27.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:27.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_41 2026-03-24T02:42:27.467 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:27.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_41 2026-03-24T02:42:27.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:27.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:27.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_41 2026-03-24T02:42:27.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_41 '' 2026-03-24T02:42:27.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:27.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:27.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_41 2026-03-24T02:42:27.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:27.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-03-24T02:42:27.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_41.status 2026-03-24T02:42:27.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:27.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:27.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:27.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-03-24T02:42:27.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:27.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:27.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:27.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:27.515 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:27.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:27.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:27.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2026-03-24T02:42:27.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:27.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:27.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_41 2026-03-24T02:42:27.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2026-03-24T02:42:27.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:27.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:27.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_41 2026-03-24T02:42:27.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:27.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-03-24T02:42:27.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_41.status 2026-03-24T02:42:27.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:27.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:27.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-03-24T02:42:27.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:27.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:27.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:27.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:27.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:27.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:27.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.status 2026-03-24T02:42:27.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:27.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:27.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:27.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_41 snap 2026-03-24T02:42:27.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:27.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:27.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:42:27.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:27.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:42:27.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:42:27.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_41@snap 2026-03-24T02:42:28.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_41': 2026-03-24T02:42:28.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:42:28.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:42:28.665 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c989d089ed 2026-03-24T02:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c989d089ed 2026-03-24T02:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:31:54 2026 2026-03-24T02:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:31:54 2026 2026-03-24T02:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:39:07 2026 2026-03-24T02:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c989d089ed 2026-03-24T02:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: cefb1221-c8b2-4bf5-a19e-5d706fe186e5 2026-03-24T02:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:42:28.669 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:28.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_41 snap 2026-03-24T02:42:28.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:28.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:42:28.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:28.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:42:28.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.export 2026-03-24T02:42:28.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_41.export 2026-03-24T02:42:28.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_41.export 2026-03-24T02:42:28.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_41@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.export 2026-03-24T02:42:28.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:28.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_41@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_41.export 2026-03-24T02:42:29.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:29.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_41.export 2026-03-24T02:42:29.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_41.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_41.export 2026-03-24T02:42:29.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:29.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:42:29.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_42 2026-03-24T02:42:29.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_42 snap 2026-03-24T02:42:29.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:29.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:29.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:42:29.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:29.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_42@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:42:30.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_42 2026-03-24T02:42:30.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:30.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:30.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:42:30.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_42 started 2026-03-24T02:42:30.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:30.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:30.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:42:30.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:42:30.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:42:30.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:30.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:31.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_42 started 2026-03-24T02:42:31.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:31.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:31.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:42:31.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:42:31.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:42:31.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:42:31.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_42 2026-03-24T02:42:31.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:42:31.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:31.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:31.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:42:31.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:42:31.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:31.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:42:31.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:42:31.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:31.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:42:31.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:31.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_42 2026-03-24T02:42:31.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:42:31.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:31.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:31.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:31.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:31.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:31.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:31.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:31.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:31.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:31.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:31.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:31.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:31.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:31.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:31.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:42:31.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:31.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:31.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:31.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:42:31.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:31.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:31.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:31.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:31.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:31.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:31.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:31.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:42:31.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:42:31.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:31.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:31.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:31.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:31.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_42 2026-03-24T02:42:31.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-03-24T02:42:31.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:42:31.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:31.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:31.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_42 2026-03-24T02:42:31.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:42:31.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:31.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:31.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:31.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:42:31.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:31.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:31.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:31.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:31.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_42 2026-03-24T02:42:31.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-03-24T02:42:31.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:42:31.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:31.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:31.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_42 2026-03-24T02:42:31.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:42:31.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:31.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:31.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:31.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:31.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:31.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:31.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:31.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:31.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:31.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:31.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:31.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:31.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:31.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:31.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:42:31.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:31.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:31.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:31.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:42:31.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:31.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:31.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:31.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:31.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:31.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:31.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:31.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:42:31.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:42:31.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:31.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:31.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:31.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:31.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_42 2026-03-24T02:42:31.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:42:31.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:42:31.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:42:31.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:42:31.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:42:31.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:42:31.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:42:31.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:31.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_42 2026-03-24T02:42:31.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:31.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:42:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:42:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_42 2026-03-24T02:42:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:42:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:42:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:42:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:42:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:31.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:42:31.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_42 2026-03-24T02:42:31.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:31.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:31.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:42:31.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:31.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_42 ']' 2026-03-24T02:42:31.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-03-24T02:42:31.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-03-24T02:42:31.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:31.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:31.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:31.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:31.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:31.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:31.879 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:31.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:31.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:31.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:31.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-03-24T02:42:31.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:31.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:31.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:31.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:31.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:31.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:31.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:31.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:31.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:31.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:31.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:31.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:31.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:31.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:31.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:31.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:31.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:31.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:31.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:31.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:31.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:31.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:31.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:31.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:31.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:31.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:31.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:31.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:31.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:31.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:31.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:31.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:31.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:31.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-03-24T02:42:31.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-03-24T02:42:31.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:31.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:31.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:31.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_42 2026-03-24T02:42:31.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:31.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:31.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:31.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:31.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:31.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:31.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:31.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:31.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:31.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:31.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:31.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:31.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:31.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:31.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:31.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:31.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:31.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:31.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:31.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:31.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:31.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:31.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:31.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:31.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:31.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:31.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:31.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:31.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:31.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:31.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:31.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:31.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:31.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_42 2026-03-24T02:42:32.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-03-24T02:42:32.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:32.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:32.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:32.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_42 2026-03-24T02:42:32.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:42:32.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:32.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:32.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:32.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:32.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:32.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:32.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_42 2026-03-24T02:42:32.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:32.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-03-24T02:42:32.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:32.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:32.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:32.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-03-24T02:42:32.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:32.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:32.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:32.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:32.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-03-24T02:42:32.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_42.status 2026-03-24T02:42:32.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:32.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:32.140 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-03-24T02:42:32.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:32.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:42:32.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:42:32.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:32.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:32.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:32.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:32.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:42:32.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-03-24T02:42:32.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:32.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:32.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:32.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-03-24T02:42:32.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:32.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:32.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:32.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:32.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-03-24T02:42:32.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_42.status 2026-03-24T02:42:32.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:32.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:32.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-03-24T02:42:32.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:32.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:42:32.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:42:32.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:32.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:32.217 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:32.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:32.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:42:32.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:42:32.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:42:32.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:42:32.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:42:32.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:42:32.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:42:32.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:42:32.226 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:32.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:42:32.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:42:32.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:42:32.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:32.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:42:32.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:42:32.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:42:32.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:32.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:42:32.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:42:32.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:42:32.232 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:32.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:42:32.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:42:32.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:42:32.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:32.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:42:32.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_42 2026-03-24T02:42:32.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:32.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:32.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:42:32.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:32.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_42 ']' 2026-03-24T02:42:32.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-03-24T02:42:32.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-03-24T02:42:32.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:32.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:32.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:32.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:32.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:32.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:32.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:32.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:32.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:32.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:32.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:32.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:32.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:32.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:32.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-03-24T02:42:32.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-03-24T02:42:32.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:32.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_42 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:32.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:32.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:32.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:32.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:32.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:32.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:32.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:32.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:32.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:32.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:32.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_42 2026-03-24T02:42:32.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-03-24T02:42:32.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:32.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_42 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:32.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:32.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:32.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:32.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:32.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:32.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:32.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:32.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:32.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:32.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:32.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:32.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:32.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:32.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:32.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_42 2026-03-24T02:42:32.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:32.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-03-24T02:42:32.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:32.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:32.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:32.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-03-24T02:42:32.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:32.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:32.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:32.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:32.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-03-24T02:42:32.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_42.status 2026-03-24T02:42:32.704 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:32.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:32.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-03-24T02:42:32.706 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:32.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:42:32.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:42:32.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:32.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:32.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:32.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:32.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:42:32.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-03-24T02:42:32.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:32.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:32.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:32.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-03-24T02:42:32.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:32.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:32.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:32.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:32.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-03-24T02:42:32.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_42.status 2026-03-24T02:42:32.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:32.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:32.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-03-24T02:42:32.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:32.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:42:32.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:42:32.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:32.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:32.983 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:32.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:32.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:42:32.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:42:32.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:42:32.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:32.989 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:42:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_42 2026-03-24T02:42:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:42:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_42 ']' 2026-03-24T02:42:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-03-24T02:42:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-03-24T02:42:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:33.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:33.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:33.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:33.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:33.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:33.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:33.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-03-24T02:42:33.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-03-24T02:42:33.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:33.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_42 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:33.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:33.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:33.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:33.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:33.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:33.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:33.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:33.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:33.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:33.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:33.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_42 2026-03-24T02:42:33.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-03-24T02:42:33.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:33.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_42 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:33.553 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:33.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:33.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:33.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:33.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:33.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:33.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:33.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:33.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:33.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:33.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:33.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:33.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:33.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:33.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_42 2026-03-24T02:42:33.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:33.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-03-24T02:42:33.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:33.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:33.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:33.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-03-24T02:42:33.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:33.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:33.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:33.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:33.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-03-24T02:42:33.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_42.status 2026-03-24T02:42:33.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:33.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:33.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-03-24T02:42:33.646 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:33.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:42:33.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:42:33.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:33.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:33.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:33.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:42:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-03-24T02:42:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-03-24T02:42:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-03-24T02:42:33.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_42.status 2026-03-24T02:42:33.687 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:33.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:33.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-03-24T02:42:33.688 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:33.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:42:33.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:42:33.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:33.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:33.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:33.725 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:33.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:42:33.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:42:33.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:42:33.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:33.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:42:34.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_42 2026-03-24T02:42:34.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:34.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:34.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:42:34.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:34.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_42 ']' 2026-03-24T02:42:34.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-03-24T02:42:34.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-03-24T02:42:34.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:34.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:34.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:34.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:34.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:34.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:34.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:34.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:34.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:34.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:34.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:34.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:34.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:34.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-03-24T02:42:34.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-03-24T02:42:34.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:34.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_42 2026-03-24T02:42:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:34.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:34.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:34.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:34.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:34.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:34.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:34.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:34.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:34.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:34.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:34.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:34.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:34.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:34.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:34.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:34.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:34.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:34.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:34.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:34.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_42 2026-03-24T02:42:34.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-03-24T02:42:34.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:34.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:34.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:34.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_42 2026-03-24T02:42:34.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:42:34.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:34.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:34.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:34.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:42:34.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:34.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:34.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:34.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:42:34.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:34.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:34.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:34.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:42:34.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:34.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:34.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:42:34.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:34.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:34.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:34.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:34.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:34.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:34.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:34.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:34.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:34.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:34.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:34.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:34.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:34.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:34.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:34.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:34.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:34.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_42 2026-03-24T02:42:34.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:34.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-03-24T02:42:34.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:34.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:34.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:34.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-03-24T02:42:34.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:34.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:34.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:34.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:34.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-03-24T02:42:34.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_42.status 2026-03-24T02:42:34.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:34.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:34.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-03-24T02:42:34.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:34.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:42:34.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:42:34.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:34.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:34.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:34.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:34.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:42:34.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-03-24T02:42:34.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:34.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:34.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:34.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-03-24T02:42:34.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:34.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:34.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:34.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:34.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-03-24T02:42:34.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_42.status 2026-03-24T02:42:34.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-03-24T02:42:34.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:34.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:42:34.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:42:34.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:34.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:34.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:34.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:34.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:42:34.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:42:34.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' 'object_number=3, tag_tid=2, entry_tid=3' 2026-03-24T02:42:34.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:34.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:42:36.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_42 2026-03-24T02:42:36.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:36.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:36.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:42:36.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:36.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_42 ']' 2026-03-24T02:42:36.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-03-24T02:42:36.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-03-24T02:42:36.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:36.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:36.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:36.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:36.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:36.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:36.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:36.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:36.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:36.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:36.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-03-24T02:42:36.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:36.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:36.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:36.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:36.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:36.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:36.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:36.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:36.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:36.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:36.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:36.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:36.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:36.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:36.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:36.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:36.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:36.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-03-24T02:42:36.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-03-24T02:42:36.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:36.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:36.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_42 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:36.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:36.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:36.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:36.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:36.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:36.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:36.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:36.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:36.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:36.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_42 2026-03-24T02:42:36.557 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5417626600052313) > weight (0.25) 2026-03-24T02:42:36.557 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-03-24T02:42:36.557 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-03-24T02:42:36.557 INFO:tasks.rbd_mirror_thrash:waiting for 5.326869282602878 secs before reviving daemons 2026-03-24T02:42:36.558 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:42:36.549+0000 7f4a51063640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 50240) UID: 1000 2026-03-24T02:42:36.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-03-24T02:42:36.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:36.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:36.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:36.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_42 2026-03-24T02:42:36.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:42:36.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:36.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:36.628 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:36.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:36.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:36.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:36.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:36.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_42 2026-03-24T02:42:36.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:36.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-03-24T02:42:36.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:36.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:36.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:36.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-03-24T02:42:36.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:36.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:36.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:36.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:36.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-03-24T02:42:36.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_42.status 2026-03-24T02:42:36.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:36.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:36.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-03-24T02:42:36.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:36.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:42:36.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:42:36.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:36.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:36.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:36.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:36.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:42:36.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-03-24T02:42:36.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:36.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:36.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:36.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-03-24T02:42:36.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:36.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:36.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:42:36.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:36.777 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-03-24T02:42:36.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_42.status 2026-03-24T02:42:36.778 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:36.779 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:36.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-03-24T02:42:36.780 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:36.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 0 2026-03-24T02:42:36.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 0 2026-03-24T02:42:36.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:36.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:36.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:36.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.status 2026-03-24T02:42:36.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:42:36.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=3, entry_tid=1' 2026-03-24T02:42:36.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:36.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_42 snap 2026-03-24T02:42:36.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:36.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:36.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:42:36.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:36.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:42:36.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:42:36.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:37.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_42@snap 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_42': 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10d11e04a2f4 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10d11e04a2f4 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:32:02 2026 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:32:02 2026 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:37:00 2026 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10d11e04a2f4 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: a2e0c846-ea47-4b56-90bc-a68c400b0302 2026-03-24T02:42:37.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:42:37.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:37.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_42 snap 2026-03-24T02:42:37.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:37.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:42:37.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:37.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:42:37.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.export 2026-03-24T02:42:37.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_42.export 2026-03-24T02:42:37.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_42.export 2026-03-24T02:42:37.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_42@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.export 2026-03-24T02:42:38.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:38.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_42@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_42.export 2026-03-24T02:42:38.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:38.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_42.export 2026-03-24T02:42:38.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_42.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_42.export 2026-03-24T02:42:38.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:38.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:42:38.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_43 2026-03-24T02:42:38.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_43 snap 2026-03-24T02:42:38.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:38.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:38.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:42:38.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:38.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_43@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:42:39.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_43 2026-03-24T02:42:39.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:39.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:39.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:42:39.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_43 started 2026-03-24T02:42:39.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:39.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:39.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:42:39.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:42:39.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:42:39.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:39.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_43 started 2026-03-24T02:42:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:42:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:42:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:42:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:42:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_43 2026-03-24T02:42:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:42:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:42:40.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:40.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:40.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:42:40.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:42:40.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:40.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:42:40.461 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:40.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:42:40.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:40.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_43 2026-03-24T02:42:40.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:42:40.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:40.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:40.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:40.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:40.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:40.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:40.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:40.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:40.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:40.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:40.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:40.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:40.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:42:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:42:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:42:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:42:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:40.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:40.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:40.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:40.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_43 2026-03-24T02:42:40.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:40.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:42:40.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:40.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:40.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_43 2026-03-24T02:42:40.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:42:40.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:40.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:40.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:40.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:40.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:40.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:40.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:40.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:40.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:40.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:40.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:40.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:40.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:42:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:42:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:42:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:42:40.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:40.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:40.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:40.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_43 2026-03-24T02:42:40.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:40.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:42:40.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:40.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:40.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_43 2026-03-24T02:42:40.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:42:40.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:40.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:40.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:40.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:40.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:40.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:40.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:40.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:40.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:40.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:40.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:40.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:40.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:40.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:40.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:42:40.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:40.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:40.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:40.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:42:40.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:40.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:40.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:40.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:40.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:40.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:40.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:40.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:42:40.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:42:40.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:40.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:40.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:40.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:40.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_43 2026-03-24T02:42:40.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:40.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:42:40.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:40.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:40.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_43 2026-03-24T02:42:40.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:3 2026-03-24T02:42:40.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:40.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:40.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:42:40.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:42:40.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:40.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:40.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:40.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:42:40.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:42:40.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:40.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:40.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:42:40.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:42:40.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:42:40.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:3 2026-03-24T02:42:40.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:40.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:40.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:42:40.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:3 2026-03-24T02:42:40.729 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:40.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:40.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:40.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:42:40.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-24T02:42:40.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:40.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:40.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=3 2026-03-24T02:42:40.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=3 2026-03-24T02:42:40.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:42:40.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:40.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:40.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:40.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_43 2026-03-24T02:42:40.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:42:40.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:42:40.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:42:40.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_43 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_43 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:42:40.820 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:42:40.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:42:40.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:40.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:42:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_43 2026-03-24T02:42:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:42:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_43 ']' 2026-03-24T02:42:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-03-24T02:42:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-03-24T02:42:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:41.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:41.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:41.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:41.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:41.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-03-24T02:42:41.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:41.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:41.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:41.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:41.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:41.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:41.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:41.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:41.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:41.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:41.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:41.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:41.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:41.024 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:41.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:41.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:41.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:41.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:41.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:41.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:41.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:41.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:41.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:41.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:41.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:41.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:41.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:41.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:41.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:41.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:41.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:41.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:41.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:41.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:41.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:41.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:41.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_43 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:41.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:41.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:41.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:41.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:41.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:41.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:41.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:41.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:41.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:41.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:41.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:41.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:41.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:41.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:41.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:41.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:41.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:41.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:41.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_43 2026-03-24T02:42:41.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:42:41.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:41.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:41.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:41.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:42:41.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:41.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:41.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:41.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:41.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:41.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:41.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:41.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:41.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:41.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:41.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:41.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_43 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:42:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:42:41.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:41.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:41.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:41.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:41.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:41.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-03-24T02:42:41.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:41.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:41.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:41.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-03-24T02:42:41.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:41.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:41.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:41.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:41.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-03-24T02:42:41.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_43.status 2026-03-24T02:42:41.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:41.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:41.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-03-24T02:42:41.377 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:41.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:41.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:41.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:41.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:41.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:41.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:41.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:42:41.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-03-24T02:42:41.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:41.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:41.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:41.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-03-24T02:42:41.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:41.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:41.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:41.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:41.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-03-24T02:42:41.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_43.status 2026-03-24T02:42:41.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:41.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:41.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-03-24T02:42:41.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:41.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:41.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:41.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:41.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:41.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:41.460 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:41.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:41.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=4, entry_tid=1' -a 'object_number=5, tag_tid=4, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:41.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=3, entry_tid=1' '!=' '' 2026-03-24T02:42:41.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:42:41.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=3, entry_tid=1' = '' 2026-03-24T02:42:41.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:41.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:42:41.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:42:41.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:41.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=4 2026-03-24T02:42:41.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:41.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:42:41.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:41.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=3 2026-03-24T02:42:41.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:42:41.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:42:41.466 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:41.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:42:41.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:41.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:42:41.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=1 2026-03-24T02:42:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 4 = 3 -a 1 -le 1 2026-03-24T02:42:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:42:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:42:41.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_43 2026-03-24T02:42:41.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:41.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:41.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:42:41.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:41.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_43 ']' 2026-03-24T02:42:41.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-03-24T02:42:41.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-03-24T02:42:41.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:41.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:41.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:41.671 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:41.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:41.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:41.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:41.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:41.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:41.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:41.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-03-24T02:42:41.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:41.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:41.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:41.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:41.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:41.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:41.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:41.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:41.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:41.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:41.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:41.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:41.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:41.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:41.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:41.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:41.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:41.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:41.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:41.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:41.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:41.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:41.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:41.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:41.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:41.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:41.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:41.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:41.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:41.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:41.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:41.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:41.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:41.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:41.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:41.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:41.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_43 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:41.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:41.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:41.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:41.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:41.781 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:41.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:41.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:41.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_43 2026-03-24T02:42:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:42:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:42:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:42:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:41.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:41.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:41.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:41.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:41.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:41.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:41.885 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-03-24T02:42:41.885 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:42:41.885 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-03-24T02:42:41.885 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-03-24T02:42:41.886 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-03-24T02:42:41.887 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-03-24T02:42:41.887 INFO:tasks.rbd_mirror_thrash:waiting for 117.46773846488304 secs before thrashing 2026-03-24T02:42:41.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:41.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:41.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:41.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_43 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:41.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:42:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:42:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:42:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:42:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:42:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:42:41.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:42:41.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:41.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:41.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:41.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:42.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:42.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-03-24T02:42:42.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:42.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:42.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:42.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-03-24T02:42:42.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:42.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:42.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:42.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:42.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-03-24T02:42:42.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_43.status 2026-03-24T02:42:42.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:42.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:42.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-03-24T02:42:42.069 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:42.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:42.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:42.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:42.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:42.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:42.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:42.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:42:42.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-03-24T02:42:42.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:42.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:42.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:42.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-03-24T02:42:42.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:42.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:42.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:42.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:42.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-03-24T02:42:42.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_43.status 2026-03-24T02:42:42.108 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:42.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:42.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-03-24T02:42:42.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:42.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:42.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:42.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:42.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:42.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:42.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:42.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:42.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=4, entry_tid=1' -a 'object_number=5, tag_tid=4, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:42.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=3, entry_tid=1' '!=' 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:42.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:42.153 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:42:42.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_43 2026-03-24T02:42:42.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:42.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:42.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:42:42.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:42.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_43 ']' 2026-03-24T02:42:42.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-03-24T02:42:42.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-03-24T02:42:42.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:42.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:42.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:42.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:42.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:42.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:42.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:42.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:42.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:42.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:42.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:42.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:42.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:42.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:42.572 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:42.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:42.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:42.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:42.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:42.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_43 2026-03-24T02:42:42.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:42.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:42.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:42.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:42.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:42.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:42.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:42.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:42.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:42.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:42.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:42.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:42.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:42.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:42.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:42.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:42.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:42.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:42.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:42.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:42.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:42.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:42.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_43 2026-03-24T02:42:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:42:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:42.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:42.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:42.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:42.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:42.736 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:42.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:42.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:42.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:42.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:42.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_43 2026-03-24T02:42:42.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:42:42.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:42.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:42.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:42:42.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:42:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:42:42.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:42:42.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:42.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:42.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:42.816 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:42.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:42.888 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-03-24T02:42:42.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:42.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:42.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:42.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-03-24T02:42:42.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:42.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:42.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:42.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:42.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-03-24T02:42:42.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_43.status 2026-03-24T02:42:42.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:42.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:42.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-03-24T02:42:42.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:42.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:42.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:42.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:42.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:42.926 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:42.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:42.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:42:42.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-03-24T02:42:42.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:42.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:42.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:42.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-03-24T02:42:42.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:42.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:42.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:42.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:42.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-03-24T02:42:42.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_43.status 2026-03-24T02:42:42.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:42.933 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:42.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-03-24T02:42:42.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:42.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:42.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:42.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:42.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:42.966 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:42.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:42.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:42.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=4, entry_tid=1' -a 'object_number=5, tag_tid=4, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:42.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=3, entry_tid=1' '!=' 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:42.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:42.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:42:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_43 2026-03-24T02:42:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:42:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_43 ']' 2026-03-24T02:42:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-03-24T02:42:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-03-24T02:42:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:43.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:43.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-03-24T02:42:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:43.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:43.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:43.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:43.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:43.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:43.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:43.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:43.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:43.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:43.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:43.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:43.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:43.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:43.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:43.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:43.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:43.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:43.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:43.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:43.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:43.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:43.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:43.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_43 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:43.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:43.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:43.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:43.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:43.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:43.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:43.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:43.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:43.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:43.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:43.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:43.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:43.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:43.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:43.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:43.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:43.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:43.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:43.875 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:43.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:43.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:43.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_43 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:43.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:42:43.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:43.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:43.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:43.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:43.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:43.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:43.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:43.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:43.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:43.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:43.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:43.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:43.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:43.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:43.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:43.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:43.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:43.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:44.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:44.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:44.025 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:44.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:44.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_43 2026-03-24T02:42:44.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:42:44.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:44.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:44.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:42:44.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:42:44.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:44.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:44.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:44.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:42:44.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:42:44.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:44.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:44.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:42:44.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:42:44.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:42:44.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:42:44.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:44.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:44.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:42:44.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:42:44.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:44.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:44.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:44.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:42:44.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:42:44.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:44.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:44.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:42:44.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:42:44.028 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:42:44.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:44.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:44.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:44.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:44.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:44.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-03-24T02:42:44.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:44.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:44.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:44.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-03-24T02:42:44.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:44.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:44.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:44.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:44.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-03-24T02:42:44.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_43.status 2026-03-24T02:42:44.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:44.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:44.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-03-24T02:42:44.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:44.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:44.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:44.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:44.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:44.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:44.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:44.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:42:44.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-03-24T02:42:44.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:44.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:44.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:44.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-03-24T02:42:44.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:44.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:44.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:44.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:44.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-03-24T02:42:44.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_43.status 2026-03-24T02:42:44.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:44.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:44.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:44.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-03-24T02:42:44.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:44.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:44.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:44.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:44.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:44.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:44.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:44.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=4, entry_tid=1' -a 'object_number=5, tag_tid=4, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:44.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=5, tag_tid=3, entry_tid=1' '!=' 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:42:44.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:44.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:42:45.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_43 2026-03-24T02:42:45.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:45.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:45.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:42:45.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:45.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_43 ']' 2026-03-24T02:42:45.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-03-24T02:42:45.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-03-24T02:42:45.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:45.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:45.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:45.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:45.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:45.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:45.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:45.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:45.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-03-24T02:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:45.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:45.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:45.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:45.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:45.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:45.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:45.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:45.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:45.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_43 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:45.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:45.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:45.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:45.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:45.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:45.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:45.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:45.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:45.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:45.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:45.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:45.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:45.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_43 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:45.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:46.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:46.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:46.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:46.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:46.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-03-24T02:42:46.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:46.068 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_43 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:3 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=3 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=3 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:3 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:46.070 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:46.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-24T02:42:46.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:3 2026-03-24T02:42:46.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:46.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:46.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:46.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=3 2026-03-24T02:42:46.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-24T02:42:46.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:46.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:46.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=3 2026-03-24T02:42:46.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=3 2026-03-24T02:42:46.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-24T02:42:46.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:46.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:46.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-24T02:42:46.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_43 2026-03-24T02:42:46.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:46.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-03-24T02:42:46.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:46.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:46.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:46.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-03-24T02:42:46.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:46.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:46.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:46.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:46.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-03-24T02:42:46.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_43.status 2026-03-24T02:42:46.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:46.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:46.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-03-24T02:42:46.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:46.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:46.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:46.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:46.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:46.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:46.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:46.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:42:46.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-03-24T02:42:46.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:46.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:46.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:46.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-03-24T02:42:46.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:46.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:46.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:42:46.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:46.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-03-24T02:42:46.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_43.status 2026-03-24T02:42:46.221 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:46.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:46.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-03-24T02:42:46.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:46.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:42:46.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:42:46.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:46.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:46.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=4, entry_tid=1], [object_number=4, tag_tid=4, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6095], [object_number=6, tag_tid=1, entry_tid=6094]]], state=connected] 2026-03-24T02:42:46.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.status 2026-03-24T02:42:46.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:42:46.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=4, entry_tid=1' -a 'object_number=5, tag_tid=4, entry_tid=1' = 'object_number=5, tag_tid=4, entry_tid=1' 2026-03-24T02:42:46.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:46.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_43 snap 2026-03-24T02:42:46.261 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:46.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:46.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:42:46.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:46.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:42:46.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:42:46.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:47.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_43@snap 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_43': 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 116dcd5a8626 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.116dcd5a8626 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:32:09 2026 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:32:09 2026 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:38:57 2026 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 116dcd5a8626 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 44c9e29c-675d-4fdd-bbe9-3d5585ac3a76 2026-03-24T02:42:47.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:42:47.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:47.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_43 snap 2026-03-24T02:42:47.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:47.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:42:47.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:47.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:42:47.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.export 2026-03-24T02:42:47.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_43.export 2026-03-24T02:42:47.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_43.export 2026-03-24T02:42:47.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_43@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.export 2026-03-24T02:42:47.486 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:47.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_43@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_43.export 2026-03-24T02:42:47.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:47.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_43.export 2026-03-24T02:42:47.741 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_43.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_43.export 2026-03-24T02:42:47.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:47.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:42:47.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_44 2026-03-24T02:42:47.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_44 snap 2026-03-24T02:42:47.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:47.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:47.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:42:47.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:47.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_44@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:42:48.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_44 2026-03-24T02:42:48.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:48.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:48.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:42:48.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_44 started 2026-03-24T02:42:48.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:48.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:48.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:42:48.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:42:48.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:42:48.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:48.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:49.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_44 started 2026-03-24T02:42:49.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:49.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:49.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:42:49.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:42:49.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:42:49.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:42:49.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_44 2026-03-24T02:42:49.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:42:49.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:49.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:49.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:42:49.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:42:49.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:49.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:42:49.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:42:49.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:49.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:42:49.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:49.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_44 2026-03-24T02:42:49.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:42:49.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:49.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:49.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:49.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:49.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:49.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:49.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:49.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:49.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:49.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:49.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:49.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:49.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:49.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:49.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:42:49.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:49.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:49.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:49.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:42:49.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:49.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:49.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:49.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:49.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:49.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:49.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:49.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:42:49.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:42:49.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:49.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:49.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:49.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:49.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_44 2026-03-24T02:42:49.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:42:49.733 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:42:49.733 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:42:49.733 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:42:49.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_44 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_44 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:49.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:42:49.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_44 2026-03-24T02:42:49.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:49.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:49.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:42:49.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:49.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_44 ']' 2026-03-24T02:42:49.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-03-24T02:42:49.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-03-24T02:42:49.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:49.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:49.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:49.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:49.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:49.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:49.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:49.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:49.937 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:49.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:49.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-03-24T02:42:49.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:49.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:49.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:49.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:49.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:49.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:49.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:49.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:49.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:49.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:49.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:49.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-03-24T02:42:50.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:50.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-03-24T02:42:50.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:50.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:50.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:50.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-03-24T02:42:50.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:50.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:50.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:50.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:50.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-03-24T02:42:50.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_44.status 2026-03-24T02:42:50.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:50.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:50.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-03-24T02:42:50.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:50.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:50.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:50.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:50.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:50.067 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:50.071 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:50.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:50.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-03-24T02:42:50.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:50.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:50.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:50.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-03-24T02:42:50.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:50.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:50.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:50.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:50.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-03-24T02:42:50.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_44.status 2026-03-24T02:42:50.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:50.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:50.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-03-24T02:42:50.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:50.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:50.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:50.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:50.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:50.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:50.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:50.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:50.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:50.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:42:50.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:42:50.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:42:50.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:50.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:50.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:42:50.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:50.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:42:50.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:50.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:42:50.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:50.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:42:50.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:50.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:42:50.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:50.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:42:50.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:50.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:42:50.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:42:50.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:42:50.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:42:50.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:42:50.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:50.124 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:42:50.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_44 2026-03-24T02:42:50.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:50.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:50.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:42:50.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:50.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_44 ']' 2026-03-24T02:42:50.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-03-24T02:42:50.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-03-24T02:42:50.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:50.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:50.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:50.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:50.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:50.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:50.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:50.326 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:50.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:50.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:50.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-03-24T02:42:50.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:50.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:50.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:50.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:50.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:50.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:50.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:50.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:50.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:50.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:50.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:50.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:50.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:50.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:50.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:50.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:50.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:50.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:50.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:50.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:50.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:50.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:50.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:50.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:50.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:50.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:50.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:50.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:50.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:50.329 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:50.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:50.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:50.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:50.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-03-24T02:42:50.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:50.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-03-24T02:42:50.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:50.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:50.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:50.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-03-24T02:42:50.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:50.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:50.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:50.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:50.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-03-24T02:42:50.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_44.status 2026-03-24T02:42:50.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:50.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:50.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-03-24T02:42:50.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:50.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:50.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:50.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:50.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:50.464 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:50.468 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-03-24T02:42:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:50.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-03-24T02:42:50.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:50.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:50.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:50.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:50.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-03-24T02:42:50.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_44.status 2026-03-24T02:42:50.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:50.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:50.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-03-24T02:42:50.472 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:50.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:50.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:50.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:50.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:50.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:50.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:50.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:50.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:50.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:50.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:50.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:42:50.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_44 2026-03-24T02:42:50.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:50.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:50.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:42:50.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:50.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_44 ']' 2026-03-24T02:42:50.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-03-24T02:42:50.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-03-24T02:42:50.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:50.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:50.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:50.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:50.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:50.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:50.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:50.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:50.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-03-24T02:42:50.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:50.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:50.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:50.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:50.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:50.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:50.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:50.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:50.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:50.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:50.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:50.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:50.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:50.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:50.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:50.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:50.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:50.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:50.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:50.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:50.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:50.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:50.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:50.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:50.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:50.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:50.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:50.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:50.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:50.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:50.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:50.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:50.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:50.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-03-24T02:42:51.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:51.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-03-24T02:42:51.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:51.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:51.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:51.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-03-24T02:42:51.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:51.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:51.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:51.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:51.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-03-24T02:42:51.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_44.status 2026-03-24T02:42:51.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:51.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:51.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-03-24T02:42:51.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:51.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:51.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:51.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:51.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:51.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:51.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:51.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:51.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-03-24T02:42:51.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:51.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:51.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:51.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-03-24T02:42:51.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:51.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:51.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:51.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:51.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-03-24T02:42:51.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_44.status 2026-03-24T02:42:51.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:51.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:51.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-03-24T02:42:51.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:51.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:51.096 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:51.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:51.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:51.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:51.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:51.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:51.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:51.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:51.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:51.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:42:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_44 2026-03-24T02:42:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:42:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_44 ']' 2026-03-24T02:42:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-03-24T02:42:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-03-24T02:42:51.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:51.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:51.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:51.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:51.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:51.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:51.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:51.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:51.904 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:51.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:51.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-03-24T02:42:51.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:51.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:51.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:51.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:51.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:51.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:51.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:51.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:51.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:51.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:51.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:51.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:51.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:51.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:51.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:51.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-03-24T02:42:52.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:52.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-03-24T02:42:52.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:52.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:52.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:52.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-03-24T02:42:52.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:52.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:52.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:52.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:52.014 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-03-24T02:42:52.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_44.status 2026-03-24T02:42:52.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:52.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:52.016 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-03-24T02:42:52.017 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:52.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:52.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:52.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:52.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:52.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:52.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:52.058 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:52.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-03-24T02:42:52.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:52.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:52.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:52.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-03-24T02:42:52.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:52.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:52.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:52.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:52.059 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-03-24T02:42:52.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:52.060 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_44.status 2026-03-24T02:42:52.061 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:52.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-03-24T02:42:52.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:42:52.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:52.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:52.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:52.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:42:52.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:52.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:42:53.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_44 2026-03-24T02:42:53.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:53.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:53.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:42:53.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:53.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_44 ']' 2026-03-24T02:42:53.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-03-24T02:42:53.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-03-24T02:42:53.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:53.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:53.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:53.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:53.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:53.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:53.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:53.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:53.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:53.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:53.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-03-24T02:42:53.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:53.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:53.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:53.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:53.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:53.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:53.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:53.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:53.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:53.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:53.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:53.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:53.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:53.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:53.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:53.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:53.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:53.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:53.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:53.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:53.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:53.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:53.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:53.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:53.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:53.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:53.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:53.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:53.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:53.908 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:53.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:53.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:53.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:53.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-03-24T02:42:54.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:54.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-03-24T02:42:54.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:54.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:54.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:54.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-03-24T02:42:54.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:54.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:54.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:54.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:54.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-03-24T02:42:54.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_44.status 2026-03-24T02:42:54.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:54.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:54.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-03-24T02:42:54.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:54.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:54.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:54.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:54.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:54.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:54.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:54.048 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:54.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-03-24T02:42:54.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:54.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:54.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:54.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-03-24T02:42:54.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:54.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:54.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:42:54.049 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:54.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-03-24T02:42:54.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_44.status 2026-03-24T02:42:54.050 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:54.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:54.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:54.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-03-24T02:42:54.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:54.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:54.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:54.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:54.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:54.090 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.status 2026-03-24T02:42:54.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:54.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:54.091 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:54.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_44 snap 2026-03-24T02:42:54.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:54.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:54.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:42:54.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:54.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:42:54.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:42:54.092 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:55.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_44@snap 2026-03-24T02:42:55.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_44': 2026-03-24T02:42:55.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:42:55.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:42:55.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:42:55.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 120974aeabfb 2026-03-24T02:42:55.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.120974aeabfb 2026-03-24T02:42:55.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:42:55.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:42:55.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:42:55.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:42:55.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:32:18 2026 2026-03-24T02:42:55.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:32:18 2026 2026-03-24T02:42:55.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:38:48 2026 2026-03-24T02:42:55.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:42:55.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 120974aeabfb 2026-03-24T02:42:55.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:42:55.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:42:55.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 6da7427c-0f9c-4d8d-8890-f684ccc387f0 2026-03-24T02:42:55.126 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:42:55.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:55.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_44 snap 2026-03-24T02:42:55.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:55.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:42:55.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:55.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:42:55.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.export 2026-03-24T02:42:55.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_44.export 2026-03-24T02:42:55.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_44.export 2026-03-24T02:42:55.131 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_44@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.export 2026-03-24T02:42:55.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:55.324 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_44@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_44.export 2026-03-24T02:42:55.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:55.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_44.export 2026-03-24T02:42:55.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_44.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_44.export 2026-03-24T02:42:55.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:55.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:42:55.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_45 2026-03-24T02:42:55.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_45 snap 2026-03-24T02:42:55.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:55.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:55.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_45 2026-03-24T02:42:55.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:55.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_45@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:42:56.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_45 2026-03-24T02:42:56.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:56.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:56.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_45 2026-03-24T02:42:56.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_45 started 2026-03-24T02:42:56.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:56.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:56.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_45 2026-03-24T02:42:56.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:42:56.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:42:56.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:56.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:57.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_45 started 2026-03-24T02:42:57.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:57.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:57.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_45 2026-03-24T02:42:57.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:42:57.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:42:57.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:42:57.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_45 2026-03-24T02:42:57.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:42:57.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:42:57.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:57.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:57.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:42:57.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:42:57.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:57.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:42:57.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:57.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:42:57.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_45 2026-03-24T02:42:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:42:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:42:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:42:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:42:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:42:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:57.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:57.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:57.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:57.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_45 2026-03-24T02:42:57.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-03-24T02:42:57.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:42:57.740 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_45 2026-03-24T02:42:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:42:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:57.742 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:57.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:57.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:57.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:57.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:57.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:57.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:57.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:42:57.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:57.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:57.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:57.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:42:57.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:57.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:57.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:57.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:57.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:57.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:57.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:57.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:42:57.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:42:57.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:57.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:57.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:57.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:57.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_45 2026-03-24T02:42:57.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-03-24T02:42:57.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:42:57.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:57.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:57.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_45 2026-03-24T02:42:57.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:42:57.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:42:57.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:57.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:57.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:57.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:42:57.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:57.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:57.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:57.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:57.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:42:57.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:57.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:57.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:57.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:57.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:42:57.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:42:57.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:42:57.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:57.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:42:57.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:42:57.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:42:57.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:42:57.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:57.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:57.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:42:57.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:42:57.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:42:57.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:42:57.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:57.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:57.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_45 2026-03-24T02:42:57.913 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:42:57.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:42:57.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:42:57.914 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:42:57.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:42:57.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_45 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_45 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_45 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_45 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:42:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:42:58.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_45 2026-03-24T02:42:58.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:58.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:58.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_45 2026-03-24T02:42:58.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:42:58.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_45 ']' 2026-03-24T02:42:58.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-03-24T02:42:58.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-03-24T02:42:58.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:42:58.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:58.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:58.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:42:58.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:42:58.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:42:58.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:42:58.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:42:58.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:42:58.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:58.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-03-24T02:42:58.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:42:58.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:58.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:58.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:42:58.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:42:58.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:58.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:58.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:58.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:42:58.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:58.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:58.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:58.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:42:58.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:58.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:42:58.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:42:58.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:58.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:58.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:42:58.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:42:58.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:58.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:58.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:58.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:42:58.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:42:58.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:58.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:58.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:42:58.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:42:58.120 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:42:58.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:58.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:58.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:42:58.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-03-24T02:42:58.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-03-24T02:42:58.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:58.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:58.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:58.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_45 2026-03-24T02:42:58.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:42:58.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:58.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:58.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:42:58.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:42:58.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:58.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:58.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:58.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:42:58.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:58.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:58.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:58.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:42:58.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:58.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:42:58.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:42:58.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:58.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:58.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:42:58.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:42:58.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:58.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:58.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:58.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:42:58.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:42:58.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:58.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:58.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:42:58.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:42:58.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:42:58.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:58.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:58.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:42:58.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_45 2026-03-24T02:42:58.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-03-24T02:42:58.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:42:58.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:42:58.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:42:58.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_45 2026-03-24T02:42:58.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:42:58.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:42:58.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:42:58.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:42:58.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:42:58.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:42:58.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:42:58.293 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:42:58.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:42:58.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:42:58.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:42:58.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:58.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:58.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:42:58.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_45 2026-03-24T02:42:58.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:58.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-03-24T02:42:58.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:58.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:58.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_45 2026-03-24T02:42:58.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-03-24T02:42:58.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:58.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:58.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_45 2026-03-24T02:42:58.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:42:58.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-03-24T02:42:58.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_45.status 2026-03-24T02:42:58.393 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:58.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_45.status 2026-03-24T02:42:58.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-03-24T02:42:58.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_45.status 2026-03-24T02:42:58.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:58.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:58.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:58.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:58.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:58.435 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_45.status 2026-03-24T02:42:58.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:58.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-03-24T02:42:58.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:58.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:58.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_45 2026-03-24T02:42:58.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-03-24T02:42:58.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:42:58.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:42:58.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_45 2026-03-24T02:42:58.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:42:58.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-03-24T02:42:58.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_45.status 2026-03-24T02:42:58.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:42:58.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_45.status 2026-03-24T02:42:58.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-03-24T02:42:58.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_45.status 2026-03-24T02:42:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:42:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:42:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:42:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:42:58.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_45.status 2026-03-24T02:42:58.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:58.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:42:58.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:58.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_45 snap 2026-03-24T02:42:58.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:42:58.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:58.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_45 2026-03-24T02:42:58.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:58.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:42:58.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:42:58.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:42:59.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_45@snap 2026-03-24T02:42:59.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_45': 2026-03-24T02:42:59.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:42:59.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:42:59.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:42:59.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c93106a64a 2026-03-24T02:42:59.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c93106a64a 2026-03-24T02:42:59.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:42:59.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:42:59.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:42:59.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:42:59.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:32:31 2026 2026-03-24T02:42:59.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:32:31 2026 2026-03-24T02:42:59.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:39:35 2026 2026-03-24T02:42:59.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:42:59.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c93106a64a 2026-03-24T02:42:59.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:42:59.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:42:59.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 91a0bb24-dfa0-431a-9655-932b666e2200 2026-03-24T02:42:59.510 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:42:59.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:42:59.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_45 snap 2026-03-24T02:42:59.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:42:59.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_45 2026-03-24T02:42:59.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:42:59.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:42:59.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_45.export 2026-03-24T02:42:59.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_45.export 2026-03-24T02:42:59.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_45.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_45.export 2026-03-24T02:42:59.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_45@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_45.export 2026-03-24T02:42:59.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:59.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_45@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_45.export 2026-03-24T02:42:59.906 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:42:59.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_45.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_45.export 2026-03-24T02:42:59.970 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_45.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_45.export 2026-03-24T02:43:00.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:00.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:00.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_46 2026-03-24T02:43:00.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_46 snap 2026-03-24T02:43:00.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:00.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:00.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:43:00.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:43:00.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_46@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:43:00.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_46 2026-03-24T02:43:00.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:00.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:00.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:43:00.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_46 started 2026-03-24T02:43:00.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:00.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:00.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:43:00.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:43:00.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:43:00.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:00.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:43:01.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_46 started 2026-03-24T02:43:01.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:01.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:43:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:43:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:43:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:43:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_46 2026-03-24T02:43:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:43:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:43:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:43:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:43:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:43:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:01.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:43:01.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:01.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:43:01.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:01.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_46 2026-03-24T02:43:01.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:43:01.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:43:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:01.695 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:01.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:01.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:01.696 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_46 2026-03-24T02:43:01.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-03-24T02:43:01.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:43:01.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:43:01.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:01.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_46 2026-03-24T02:43:01.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:43:01.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:43:01.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:01.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:43:01.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:43:01.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:43:01.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:01.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:01.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:43:01.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:43:01.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:43:01.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:43:01.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:43:01.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:43:01.773 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:43:01.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:43:01.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:43:01.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:43:01.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:43:01.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:43:01.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:43:01.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:43:01.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:43:01.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:43:01.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:43:01.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:43:01.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:43:01.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:43:01.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:43:01.774 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:43:01.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:43:01.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:43:01.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:43:01.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_46 2026-03-24T02:43:01.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-03-24T02:43:01.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:43:01.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:43:01.866 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_46 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:2 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:2 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:43:01.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:2 2026-03-24T02:43:01.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:43:01.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:43:01.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:43:01.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:43:01.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-24T02:43:01.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:43:01.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:43:01.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=2 2026-03-24T02:43:01.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=2 2026-03-24T02:43:01.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:43:01.883 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:43:01.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:43:01.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:43:01.884 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_46 2026-03-24T02:43:01.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:43:01.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:43:01.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:43:01.967 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:43:01.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_46 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_46 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:43:01.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:43:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_46 2026-03-24T02:43:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:43:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:43:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_46 ']' 2026-03-24T02:43:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-03-24T02:43:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-03-24T02:43:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:43:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:43:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:43:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:02.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:02.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:02.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:02.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:02.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:02.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:02.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:02.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:02.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:02.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:02.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:02.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:02.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:02.174 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:02.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:02.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:02.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:02.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-03-24T02:43:02.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-03-24T02:43:02.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:43:02.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:43:02.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:02.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-03-24T02:43:02.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:43:02.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:02.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:02.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:43:02.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:43:02.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:02.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:02.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:02.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:43:02.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:43:02.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:02.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:02.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:43:02.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:43:02.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:43:02.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:43:02.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:02.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:02.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:43:02.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:43:02.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:02.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:02.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:02.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:43:02.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:43:02.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:02.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:02.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:43:02.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:43:02.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:43:02.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:43:02.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:43:02.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:43:02.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-03-24T02:43:02.343 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-03-24T02:43:02.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:43:02.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:43:02.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:02.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:43:02.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:43:02.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:43:02.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:43:02.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:43:02.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2026-03-24T02:43:02.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:02.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-03-24T02:43:02.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:02.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:02.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_46 2026-03-24T02:43:02.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-03-24T02:43:02.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:02.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:02.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_46 2026-03-24T02:43:02.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:43:02.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-03-24T02:43:02.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_46.status 2026-03-24T02:43:02.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:02.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_46.status 2026-03-24T02:43:02.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-03-24T02:43:02.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_46.status 2026-03-24T02:43:02.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:43:02.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:43:02.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:02.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:02.477 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:02.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_46.status 2026-03-24T02:43:02.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:43:02.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-03-24T02:43:02.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:02.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:02.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_46 2026-03-24T02:43:02.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-03-24T02:43:02.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:02.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:02.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_46 2026-03-24T02:43:02.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:43:02.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-03-24T02:43:02.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_46.status 2026-03-24T02:43:02.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:02.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_46.status 2026-03-24T02:43:02.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-03-24T02:43:02.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_46.status 2026-03-24T02:43:02.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:43:02.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:43:02.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:02.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:02.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:02.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_46.status 2026-03-24T02:43:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:43:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:43:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:43:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:43:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:43:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:43:02.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:43:02.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:43:02.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:43:02.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:43:02.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:43:02.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:43:02.526 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:43:02.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:43:02.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:43:02.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:43:02.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:43:02.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:43:02.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:43:02.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:43:02.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:43:02.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:43:02.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:43:02.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:43:02.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:43:02.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:43:02.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_46 2026-03-24T02:43:02.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:02.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:02.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:43:02.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:43:02.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_46 ']' 2026-03-24T02:43:02.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-03-24T02:43:02.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-03-24T02:43:02.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:43:02.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:02.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:02.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:43:02.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:43:02.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:02.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:02.733 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:02.733 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:43:02.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:02.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-03-24T02:43:02.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:43:02.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:02.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:02.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:02.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:43:02.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:02.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:02.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:02.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:02.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:02.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:02.752 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-03-24T02:43:02.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-03-24T02:43:02.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:43:02.819 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:43:02.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:02.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:02.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:43:02.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:43:02.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:43:02.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:43:02.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:43:02.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:43:02.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-03-24T02:43:02.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-03-24T02:43:02.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:43:02.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:2 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=2 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=2 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:2 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:2 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=2 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=2 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=2 2026-03-24T02:43:02.901 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-24T02:43:02.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:43:02.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:43:02.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-24T02:43:02.915 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2026-03-24T02:43:02.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:02.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-03-24T02:43:02.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:02.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:02.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_46 2026-03-24T02:43:02.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-03-24T02:43:02.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:02.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:02.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_46 2026-03-24T02:43:02.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:43:02.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-03-24T02:43:02.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_46.status 2026-03-24T02:43:02.994 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:02.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_46.status 2026-03-24T02:43:02.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-03-24T02:43:02.995 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_46.status 2026-03-24T02:43:03.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:43:03.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:43:03.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:03.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:03.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_46.status 2026-03-24T02:43:03.036 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:43:03.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-03-24T02:43:03.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:03.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:03.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_46 2026-03-24T02:43:03.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-03-24T02:43:03.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:03.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:03.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_46 2026-03-24T02:43:03.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:43:03.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-03-24T02:43:03.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_46.status 2026-03-24T02:43:03.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:03.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_46.status 2026-03-24T02:43:03.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-03-24T02:43:03.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_46.status 2026-03-24T02:43:03.073 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:43:03.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:43:03.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:03.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:03.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:03.077 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_46.status 2026-03-24T02:43:03.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:43:03.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:43:03.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:03.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_46 snap 2026-03-24T02:43:03.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:03.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:03.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:43:03.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:43:03.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:43:03.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:43:03.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:43:04.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_46@snap 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_46': 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10d1d40354f8 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10d1d40354f8 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:32:45 2026 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:32:45 2026 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:39:21 2026 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10d1d40354f8 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 2c7365e1-7047-43b0-b5bd-9425144b8319 2026-03-24T02:43:04.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:43:04.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:04.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_46 snap 2026-03-24T02:43:04.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:04.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:43:04.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:43:04.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:43:04.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_46.export 2026-03-24T02:43:04.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_46.export 2026-03-24T02:43:04.116 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_46.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_46.export 2026-03-24T02:43:04.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_46@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_46.export 2026-03-24T02:43:04.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:43:04.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_46@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_46.export 2026-03-24T02:43:04.529 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:43:04.539 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_46.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_46.export 2026-03-24T02:43:04.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_46.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_46.export 2026-03-24T02:43:04.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:04.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:04.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_47 2026-03-24T02:43:04.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_47 snap 2026-03-24T02:43:04.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:04.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:04.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:43:04.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:43:04.638 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_47@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:43:05.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_47 2026-03-24T02:43:05.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:05.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:05.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:43:05.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_47 started 2026-03-24T02:43:05.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:05.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:05.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:43:05.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:43:05.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:43:05.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:05.842 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:43:06.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_47 started 2026-03-24T02:43:06.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:06.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:06.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:43:06.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:43:06.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:43:06.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:43:06.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_47 2026-03-24T02:43:06.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:43:06.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:43:06.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:43:06.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:06.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:43:06.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:43:06.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:06.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:43:06.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:06.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:43:06.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:06.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_47 2026-03-24T02:43:06.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:43:06.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:43:06.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:06.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:06.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:43:06.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:43:06.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:06.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:06.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:06.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:06.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_47 2026-03-24T02:43:06.938 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:43:06.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:43:06.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:43:06.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:43:06.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_47 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_47 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:43:06.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:43:07.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_47 2026-03-24T02:43:07.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:07.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:07.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:43:07.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:43:07.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_47 ']' 2026-03-24T02:43:07.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_47' 2026-03-24T02:43:07.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2026-03-24T02:43:07.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:43:07.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:07.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:07.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:43:07.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:43:07.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:07.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:07.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:07.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:43:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2026-03-24T02:43:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:43:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:43:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:43:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:43:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:07.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:07.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:43:07.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:07.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:07.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:07.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:07.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:07.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:07.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:07.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:07.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:07.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:07.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:07.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:07.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:07.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:07.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:07.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:07.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:07.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2026-03-24T02:43:07.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:07.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_47 2026-03-24T02:43:07.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:07.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:07.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_47 2026-03-24T02:43:07.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_47 '' 2026-03-24T02:43:07.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:07.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:07.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_47 2026-03-24T02:43:07.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:43:07.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-03-24T02:43:07.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_47.status 2026-03-24T02:43:07.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:07.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:07.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-03-24T02:43:07.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:07.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:43:07.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:43:07.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:07.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:07.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:07.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:07.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:43:07.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2026-03-24T02:43:07.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:07.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:07.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_47 2026-03-24T02:43:07.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2026-03-24T02:43:07.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:07.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:07.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_47 2026-03-24T02:43:07.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:43:07.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-03-24T02:43:07.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_47.status 2026-03-24T02:43:07.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:07.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:07.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-03-24T02:43:07.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:07.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:43:07.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:43:07.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:07.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:07.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:07.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:07.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:43:07.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:43:07.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:43:07.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:43:07.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:43:07.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:43:07.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:43:07.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:43:07.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:43:07.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:43:07.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:43:07.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:43:07.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:43:07.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:43:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:43:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:43:07.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:43:07.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:43:07.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:43:07.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:43:07.341 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:43:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:43:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:43:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:43:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:43:07.342 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:43:07.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_47 2026-03-24T02:43:07.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:07.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:07.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:43:07.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:43:07.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_47 ']' 2026-03-24T02:43:07.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_47' 2026-03-24T02:43:07.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2026-03-24T02:43:07.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:43:07.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:07.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:07.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:43:07.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:43:07.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:07.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:07.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:07.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:43:07.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:07.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2026-03-24T02:43:07.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:43:07.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:07.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:07.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:07.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:43:07.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:07.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:07.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:07.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:43:07.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:07.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:07.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:07.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:07.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:07.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:07.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:07.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2026-03-24T02:43:07.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:07.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_47 2026-03-24T02:43:07.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:07.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:07.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_47 2026-03-24T02:43:07.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_47 '' 2026-03-24T02:43:07.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:07.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:07.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_47 2026-03-24T02:43:07.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:43:07.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-03-24T02:43:07.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_47.status 2026-03-24T02:43:07.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:07.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:07.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-03-24T02:43:07.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:07.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:43:07.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:43:07.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:07.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:07.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:07.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2026-03-24T02:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_47 2026-03-24T02:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2026-03-24T02:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_47 2026-03-24T02:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-03-24T02:43:07.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_47.status 2026-03-24T02:43:07.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:07.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:07.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-03-24T02:43:07.718 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:07.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:43:07.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:43:07.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:07.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:07.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2], [object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:07.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:07.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:43:07.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:43:07.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=3' '!=' 'object_number=11, tag_tid=2, entry_tid=3' 2026-03-24T02:43:07.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:43:07.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:43:08.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_47 2026-03-24T02:43:08.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:08.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:08.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:43:08.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:43:08.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_47 ']' 2026-03-24T02:43:08.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_47' 2026-03-24T02:43:08.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2026-03-24T02:43:08.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:43:08.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:08.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:08.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:43:08.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:43:08.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:08.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:08.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:08.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:43:08.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:08.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2026-03-24T02:43:08.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:43:08.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:08.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:08.170 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2026-03-24T02:43:08.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:08.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_47 2026-03-24T02:43:08.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:08.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:08.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_47 2026-03-24T02:43:08.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_47 '' 2026-03-24T02:43:08.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:08.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:08.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_47 2026-03-24T02:43:08.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:43:08.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-03-24T02:43:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_47.status 2026-03-24T02:43:08.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:08.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:08.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-03-24T02:43:08.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:08.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:43:08.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:43:08.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:08.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:08.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:08.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:08.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:43:08.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2026-03-24T02:43:08.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:08.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:08.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_47 2026-03-24T02:43:08.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2026-03-24T02:43:08.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:08.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:08.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_47 2026-03-24T02:43:08.314 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:43:08.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-03-24T02:43:08.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_47.status 2026-03-24T02:43:08.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:08.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:08.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-03-24T02:43:08.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:08.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 2 2026-03-24T02:43:08.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 2 2026-03-24T02:43:08.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:08.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:08.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=9, tag_tid=3, entry_tid=1], [object_number=8, tag_tid=3, entry_tid=0], [object_number=11, tag_tid=2, entry_tid=3], [object_number=10, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:08.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.status 2026-03-24T02:43:08.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:43:08.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=9, tag_tid=3, entry_tid=1' -a 'object_number=9, tag_tid=3, entry_tid=1' = 'object_number=9, tag_tid=3, entry_tid=1' 2026-03-24T02:43:08.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:08.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_47 snap 2026-03-24T02:43:08.355 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:43:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:43:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:43:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:43:08.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:43:09.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_47@snap 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_47': 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 116d38186b86 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.116d38186b86 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:33:01 2026 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:33:01 2026 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:39:18 2026 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 116d38186b86 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 2c7ddb9c-7dd2-4420-8c5c-f002acffd503 2026-03-24T02:43:09.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:43:09.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:09.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_47 snap 2026-03-24T02:43:09.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:09.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:43:09.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:43:09.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:43:09.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.export 2026-03-24T02:43:09.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_47.export 2026-03-24T02:43:09.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_47.export 2026-03-24T02:43:09.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_47@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.export 2026-03-24T02:43:09.574 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:43:09.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_47@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_47.export 2026-03-24T02:43:09.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:43:09.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_47.export 2026-03-24T02:43:09.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_47.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_47.export 2026-03-24T02:43:09.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:09.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:09.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_48 2026-03-24T02:43:09.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_48 snap 2026-03-24T02:43:09.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:09.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:09.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_48 2026-03-24T02:43:09.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:43:09.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_48@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:43:10.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_48 2026-03-24T02:43:10.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:10.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:10.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_48 2026-03-24T02:43:10.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_48 started 2026-03-24T02:43:10.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:10.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:10.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_48 2026-03-24T02:43:10.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:43:10.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:43:10.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:10.762 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:43:11.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_48 started 2026-03-24T02:43:11.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:11.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:11.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_48 2026-03-24T02:43:11.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:43:11.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:43:11.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:43:11.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_48 2026-03-24T02:43:11.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:43:11.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:43:11.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:43:11.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:11.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:43:11.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:43:11.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:11.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:43:11.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:11.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:43:11.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:11.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_48 2026-03-24T02:43:11.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:43:11.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:11.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:11.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:43:11.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:43:11.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:43:11.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:43:11.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:11.783 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:11.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:11.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:11.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_48 2026-03-24T02:43:11.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-03-24T02:43:11.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror status : get status for rbd mirror 2026-03-24T02:43:11.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:43:11.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:11.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_48 2026-03-24T02:43:11.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:1 2026-03-24T02:43:11.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:43:11.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:11.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:43:11.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:43:11.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:43:11.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:11.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:11.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:43:11.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:43:11.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:1 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:1 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=1 2026-03-24T02:43:11.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=1 2026-03-24T02:43:11.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:43:11.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:43:11.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:43:11.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:43:11.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_48 2026-03-24T02:43:11.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:43:11.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:43:11.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:43:11.953 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:43:11.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:43:11.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:43:11.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:43:11.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:11.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_48 2026-03-24T02:43:11.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:43:11.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:11.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:43:11.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:43:11.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_48 2026-03-24T02:43:11.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:43:11.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_48 2026-03-24T02:43:11.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:43:11.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:11.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:43:11.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:43:11.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_48 2026-03-24T02:43:11.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:43:11.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:43:11.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:43:11.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:43:11.955 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:43:12.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_48 2026-03-24T02:43:12.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:12.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:12.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_48 2026-03-24T02:43:12.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:43:12.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_48 ']' 2026-03-24T02:43:12.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_48' 2026-03-24T02:43:12.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_48 2026-03-24T02:43:12.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:43:12.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:12.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:12.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:43:12.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:43:12.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:12.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:12.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:12.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_48 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:12.158 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:12.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:12.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:12.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:12.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:12.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:12.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:12.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:12.159 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:12.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:12.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:12.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:12.172 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_48 2026-03-24T02:43:12.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-03-24T02:43:12.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd mirror flush : flush rbd mirror 2026-03-24T02:43:12.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:admin_socket: invalid command 2026-03-24T02:43:12.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:12.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_48 2026-03-24T02:43:12.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:43:12.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:12.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:12.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-24T02:43:12.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:1 2026-03-24T02:43:12.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:12.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=1 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=1 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:1 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:1 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=1 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=1 2026-03-24T02:43:12.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=1 2026-03-24T02:43:12.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-24T02:43:12.256 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:43:12.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:43:12.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-24T02:43:12.257 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_48 2026-03-24T02:43:12.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:12.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_48 2026-03-24T02:43:12.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:12.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:12.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_48 2026-03-24T02:43:12.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_48 '' 2026-03-24T02:43:12.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:12.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:12.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_48 2026-03-24T02:43:12.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:43:12.336 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-03-24T02:43:12.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_48.status 2026-03-24T02:43:12.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:12.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_48.status 2026-03-24T02:43:12.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-03-24T02:43:12.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_48.status 2026-03-24T02:43:12.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:12.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:12.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:12.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:12.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:12.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_48.status 2026-03-24T02:43:12.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:12.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_48 2026-03-24T02:43:12.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:12.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:12.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_48 2026-03-24T02:43:12.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_48 '..*' 2026-03-24T02:43:12.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:12.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:12.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_48 2026-03-24T02:43:12.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:43:12.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-03-24T02:43:12.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_48.status 2026-03-24T02:43:12.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:12.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_48.status 2026-03-24T02:43:12.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-03-24T02:43:12.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_48.status 2026-03-24T02:43:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:12.623 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_48.status 2026-03-24T02:43:12.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:12.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:12.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:12.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_48 snap 2026-03-24T02:43:12.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:12.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:12.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_48 2026-03-24T02:43:12.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:43:12.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:43:12.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:43:12.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:43:13.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_48@snap 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_48': 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 12094621d5cf 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.12094621d5cf 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:33:13 2026 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:33:13 2026 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:38:35 2026 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 12094621d5cf 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: e57f0e8e-4e30-48f9-b541-61ba6417a6ad 2026-03-24T02:43:13.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:43:13.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:13.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_48 snap 2026-03-24T02:43:13.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:13.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_48 2026-03-24T02:43:13.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:43:13.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:43:13.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_48.export 2026-03-24T02:43:13.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_48.export 2026-03-24T02:43:13.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_48.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_48.export 2026-03-24T02:43:13.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_48@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_48.export 2026-03-24T02:43:13.844 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:43:13.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_48@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_48.export 2026-03-24T02:43:14.054 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:43:14.064 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_48.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_48.export 2026-03-24T02:43:14.119 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_48.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_48.export 2026-03-24T02:43:14.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:14.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:14.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_49 2026-03-24T02:43:14.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_49 snap 2026-03-24T02:43:14.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:14.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:14.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:43:14.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:43:14.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_49@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:43:14.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_49 2026-03-24T02:43:14.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:14.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:14.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:43:14.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_49 started 2026-03-24T02:43:14.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:14.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:14.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:43:14.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:43:14.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:43:14.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:14.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:43:15.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_49 started 2026-03-24T02:43:15.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:15.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:15.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:43:15.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:43:15.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:43:15.786 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:43:15.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_49 2026-03-24T02:43:15.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:43:15.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:43:15.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:43:15.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:15.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:43:15.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:43:15.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:15.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:43:15.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:15.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:43:15.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:15.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_49 2026-03-24T02:43:15.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:43:15.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:43:15.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:15.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:15.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:15.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:43:15.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:15.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:15.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:43:15.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:43:15.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:15.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:15.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:15.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:15.805 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_49 2026-03-24T02:43:15.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:43:15.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:43:15.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:43:15.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:43:15.895 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_49 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_49 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:43:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:43:16.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_49 2026-03-24T02:43:16.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:16.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:16.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:43:16.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:43:16.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_49 ']' 2026-03-24T02:43:16.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-03-24T02:43:16.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-03-24T02:43:16.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:43:16.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:16.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:16.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:43:16.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:43:16.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:16.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:16.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:16.098 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:43:16.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:16.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-03-24T02:43:16.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:43:16.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:16.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:16.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:16.099 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:16.100 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:16.117 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:16.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:16.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:16.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-03-24T02:43:16.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:16.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-03-24T02:43:16.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:16.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:16.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:16.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-03-24T02:43:16.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:16.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:16.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:16.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:43:16.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-03-24T02:43:16.200 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_49.status 2026-03-24T02:43:16.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:16.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:16.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-03-24T02:43:16.202 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:16.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:16.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:16.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:16.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:16.233 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:16.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:16.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-03-24T02:43:16.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:16.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-03-24T02:43:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:43:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-03-24T02:43:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_49.status 2026-03-24T02:43:16.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:16.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:16.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-03-24T02:43:16.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:16.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:16.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:16.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:16.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:16.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:16.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:16.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:16.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:43:16.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:43:16.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:43:16.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:16.285 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:16.286 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:43:16.286 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:43:16.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:43:16.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:16.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:43:16.288 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:43:16.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:43:16.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:16.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:43:16.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:43:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:43:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:43:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:43:16.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:43:16.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:43:16.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:43:16.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:43:16.294 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:43:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_49 2026-03-24T02:43:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:43:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:43:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_49 ']' 2026-03-24T02:43:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-03-24T02:43:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-03-24T02:43:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:43:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:43:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:43:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:16.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:16.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:16.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:16.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:16.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:16.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-03-24T02:43:16.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:16.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-03-24T02:43:16.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:16.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:16.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:16.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-03-24T02:43:16.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:16.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:16.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:16.595 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:43:16.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-03-24T02:43:16.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_49.status 2026-03-24T02:43:16.596 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:16.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:16.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-03-24T02:43:16.598 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:16.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:16.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:16.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:16.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:16.841 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:16.845 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:16.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:16.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-03-24T02:43:16.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:16.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:16.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:16.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-03-24T02:43:16.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:16.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:16.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:16.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:43:16.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-03-24T02:43:16.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_49.status 2026-03-24T02:43:16.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:16.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:16.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-03-24T02:43:16.849 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:16.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:16.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:16.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:16.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:16.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:16.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:16.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:16.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:16.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:16.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:43:16.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:43:17.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_49 2026-03-24T02:43:17.295 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:43:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:43:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_49 ']' 2026-03-24T02:43:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-03-24T02:43:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-03-24T02:43:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:43:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:43:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:43:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:43:17.297 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:17.298 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:17.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:17.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:17.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:17.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:17.299 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:17.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:17.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:17.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:17.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-03-24T02:43:17.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:17.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-03-24T02:43:17.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:17.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:17.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:17.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-03-24T02:43:17.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:17.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:17.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:17.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:43:17.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-03-24T02:43:17.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_49.status 2026-03-24T02:43:17.399 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:17.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:17.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-03-24T02:43:17.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:17.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:17.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:17.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:17.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:17.437 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:17.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:17.442 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:17.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-03-24T02:43:17.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:17.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:17.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:17.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-03-24T02:43:17.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:17.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:17.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:17.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:43:17.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-03-24T02:43:17.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_49.status 2026-03-24T02:43:17.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:17.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:17.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-03-24T02:43:17.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:17.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:17.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:17.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:17.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:17.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:17.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:17.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:17.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:17.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:17.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:43:17.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.8 2026-03-24T02:43:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_49 2026-03-24T02:43:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:43:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:43:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_49 ']' 2026-03-24T02:43:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-03-24T02:43:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-03-24T02:43:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:43:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:43:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:43:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:18.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:43:18.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:18.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-03-24T02:43:18.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:43:18.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:18.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:18.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:18.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:18.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:18.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:18.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:18.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-03-24T02:43:18.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:18.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-03-24T02:43:18.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:18.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:18.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:18.391 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-03-24T02:43:18.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:18.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:18.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:18.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:43:18.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-03-24T02:43:18.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:18.392 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_49.status 2026-03-24T02:43:18.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:18.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-03-24T02:43:18.394 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:18.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:18.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:18.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:18.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:18.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:18.436 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-03-24T02:43:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-03-24T02:43:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:18.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:43:18.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-03-24T02:43:18.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_49.status 2026-03-24T02:43:18.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:18.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:18.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-03-24T02:43:18.441 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:18.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:18.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:18.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:18.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:18.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:18.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:18.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:18.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:18.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:18.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:43:18.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1.6 2026-03-24T02:43:20.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_49 2026-03-24T02:43:20.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:20.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:20.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:43:20.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:43:20.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_49 ']' 2026-03-24T02:43:20.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-03-24T02:43:20.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-03-24T02:43:20.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:43:20.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:20.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:20.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:43:20.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:43:20.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:20.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:20.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:20.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:43:20.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:20.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-03-24T02:43:20.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:43:20.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:20.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:20.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:20.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:43:20.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:20.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:20.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:20.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:20.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:20.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:20.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:20.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:20.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-03-24T02:43:20.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:20.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-03-24T02:43:20.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:20.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:20.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:20.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-03-24T02:43:20.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:20.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:20.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:20.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:43:20.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-03-24T02:43:20.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_49.status 2026-03-24T02:43:20.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:20.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:20.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-03-24T02:43:20.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:20.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:20.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:20.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:20.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:20.216 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:20.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:20.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:20.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-03-24T02:43:20.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:20.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:20.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:20.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-03-24T02:43:20.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:20.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:20.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:20.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:43:20.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-03-24T02:43:20.222 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_49.status 2026-03-24T02:43:20.223 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:20.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:20.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-03-24T02:43:20.225 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:20.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:20.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:20.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:20.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:20.264 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:20.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:20.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:20.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:20.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:20.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:43:20.270 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 2 2026-03-24T02:43:22.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_49 2026-03-24T02:43:22.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:22.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:22.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:43:22.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:43:22.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_49 ']' 2026-03-24T02:43:22.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-03-24T02:43:22.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-03-24T02:43:22.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:43:22.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:22.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:22.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:43:22.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:43:22.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:22.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:22.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:22.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:43:22.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:22.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-03-24T02:43:22.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:43:22.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:22.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:22.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:22.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:43:22.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:22.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:22.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:22.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:22.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:22.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:22.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:22.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:22.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:22.289 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-03-24T02:43:22.366 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:22.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-03-24T02:43:22.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:22.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:22.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:22.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-03-24T02:43:22.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:22.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:22.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:22.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:43:22.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-03-24T02:43:22.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_49.status 2026-03-24T02:43:22.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-03-24T02:43:22.370 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:22.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:22.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:22.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:22.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:22.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:22.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:22.412 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:22.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-03-24T02:43:22.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:22.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:22.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:22.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-03-24T02:43:22.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:22.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:22.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:43:22.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:43:22.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-03-24T02:43:22.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_49.status 2026-03-24T02:43:22.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:22.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:22.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-03-24T02:43:22.415 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:22.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:22.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:22.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:22.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:22.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:22.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.status 2026-03-24T02:43:22.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:22.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:22.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:22.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_49 snap 2026-03-24T02:43:22.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:22.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:22.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:43:22.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:43:22.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:43:22.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:43:22.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:43:23.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_49@snap 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_49': 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10c96e9c5971 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10c96e9c5971 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:33:27 2026 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:33:27 2026 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:38:22 2026 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10c96e9c5971 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 8c4ef25d-841d-488a-867a-e01b5bd9e206 2026-03-24T02:43:23.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:43:23.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:23.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_49 snap 2026-03-24T02:43:23.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:23.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:43:23.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:43:23.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:43:23.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.export 2026-03-24T02:43:23.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_49.export 2026-03-24T02:43:23.499 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_49.export 2026-03-24T02:43:23.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_49@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.export 2026-03-24T02:43:23.702 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:43:23.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_49@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_49.export 2026-03-24T02:43:23.932 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:43:23.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_49.export 2026-03-24T02:43:24.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_49.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_49.export 2026-03-24T02:43:24.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:24.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:24.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_50 2026-03-24T02:43:24.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_snap cluster2 mirror image_50 snap 2026-03-24T02:43:24.052 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:24.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:24.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:43:24.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:43:24.053 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror snap create image_50@snap --debug-rbd=20 --debug-journaler=20 2026-03-24T02:43:24.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_started cluster1 mirror image_50 2026-03-24T02:43:24.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:24.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:24.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:43:24.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_replay_state cluster1 mirror image_50 started 2026-03-24T02:43:24.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:24.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:24.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:43:24.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=started 2026-03-24T02:43:24.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:43:24.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:24.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:43:25.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_replay_state cluster1 mirror image_50 started 2026-03-24T02:43:25.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:25.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:25.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:43:25.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=started 2026-03-24T02:43:25.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local status_result 2026-03-24T02:43:25.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=stopped 2026-03-24T02:43:25.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_50 2026-03-24T02:43:25.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -i state 2026-03-24T02:43:25.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_instance=cluster1 2026-03-24T02:43:25.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:43:25.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:25.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance=cluster1 2026-03-24T02:43:25.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local loop_instance 2026-03-24T02:43:25.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:25.921 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sleep 0 2026-03-24T02:43:25.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:25.922 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ seq 0 3 2026-03-24T02:43:25.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:25.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_50 2026-03-24T02:43:25.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1:0 2026-03-24T02:43:25.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ shift 2026-03-24T02:43:25.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:25.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:25.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:25.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=cluster 2026-03-24T02:43:25.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:25.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:25.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:25.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:25.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval cluster=cluster1 2026-03-24T02:43:25.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:43:25.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:25.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:25.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:25.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local local_cluster=cluster1:0 2026-03-24T02:43:25.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster=cluster1 2026-03-24T02:43:25.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance 2026-03-24T02:43:25.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:25.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local val=cluster1:0 2026-03-24T02:43:25.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local cluster_var_name=local_cluster 2026-03-24T02:43:25.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local instance_var_name=instance 2026-03-24T02:43:25.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ cluster=cluster1 2026-03-24T02:43:25.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:25.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:25.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval local_cluster=cluster1 2026-03-24T02:43:25.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ local_cluster=cluster1 2026-03-24T02:43:25.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ eval instance=0 2026-03-24T02:43:25.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ instance=0 2026-03-24T02:43:25.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:25.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:25.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:25.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:25.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_50 2026-03-24T02:43:26.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ return 0 2026-03-24T02:43:26.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ status_result=' "state": "Replaying"' 2026-03-24T02:43:26.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo ' "state": "Replaying"' 2026-03-24T02:43:26.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ grep -i Replaying 2026-03-24T02:43:26.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: "state": "Replaying" 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=started 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test started = started 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_50 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' journal = journal ']' 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_50 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_cluster=cluster1 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local local_pool=mirror 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local remote_pool=mirror 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:43:26.033 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:43:26.234 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_50 2026-03-24T02:43:26.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:26.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:26.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:43:26.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:43:26.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_50 ']' 2026-03-24T02:43:26.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_50' 2026-03-24T02:43:26.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_50 2026-03-24T02:43:26.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:43:26.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:26.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:26.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:43:26.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:43:26.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:26.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:26.235 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:26.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_50 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:26.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:26.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:26.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:26.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:26.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:26.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:26.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:26.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:26.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:26.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:26.252 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_50 2026-03-24T02:43:26.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:26.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_50 2026-03-24T02:43:26.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:26.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:26.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_50 2026-03-24T02:43:26.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_50 '' 2026-03-24T02:43:26.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:26.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:26.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_50 2026-03-24T02:43:26.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:43:26.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-03-24T02:43:26.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_50.status 2026-03-24T02:43:26.332 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:26.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:26.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-03-24T02:43:26.333 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:26.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:26.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:26.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:26.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:26.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:26.371 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:26.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_50 2026-03-24T02:43:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_50 2026-03-24T02:43:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_50 '..*' 2026-03-24T02:43:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_50 2026-03-24T02:43:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:43:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-03-24T02:43:26.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_50.status 2026-03-24T02:43:26.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:26.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:26.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-03-24T02:43:26.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:26.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:26.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:26.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:26.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:26.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:26.417 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:26.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:26.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' '' 2026-03-24T02:43:26.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ break 2026-03-24T02:43:26.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' = '' 2026-03-24T02:43:26.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ last_mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:26.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:26.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:43:26.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:43:26.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_tag=3 2026-03-24T02:43:26.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:26.420 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-03-24T02:43:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:43:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_tag=2 2026-03-24T02:43:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:26.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:43:26.423 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:43:26.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_entry=1 2026-03-24T02:43:26.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:26.425 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-03-24T02:43:26.426 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut -d= -f 2 2026-03-24T02:43:26.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_entry=3 2026-03-24T02:43:26.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 3 = 2 -a 1 -le 3 2026-03-24T02:43:26.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:43:26.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:43:26.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.2 2026-03-24T02:43:26.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_50 2026-03-24T02:43:26.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:26.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:26.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:43:26.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:43:26.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_50 ']' 2026-03-24T02:43:26.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_50' 2026-03-24T02:43:26.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_50 2026-03-24T02:43:26.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:43:26.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:26.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:26.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:43:26.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:43:26.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:26.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:26.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:26.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:43:26.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:26.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_50 2026-03-24T02:43:26.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:43:26.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:26.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:26.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:26.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:43:26.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:26.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:26.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:26.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:43:26.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:26.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:26.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:26.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:43:26.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:26.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:26.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:43:26.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:26.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:26.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:26.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:26.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:26.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:26.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:26.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:26.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:26.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:26.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:26.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:26.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:26.632 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:26.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:26.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:26.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:26.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_50 2026-03-24T02:43:26.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:26.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_50 2026-03-24T02:43:26.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:26.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:26.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_50 2026-03-24T02:43:26.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_50 '' 2026-03-24T02:43:26.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:26.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:26.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_50 2026-03-24T02:43:26.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:43:26.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-03-24T02:43:26.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_50.status 2026-03-24T02:43:26.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:26.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:26.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-03-24T02:43:26.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:26.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:26.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:26.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:26.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:26.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:26.766 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:26.767 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:26.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_50 2026-03-24T02:43:26.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:26.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:26.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_50 2026-03-24T02:43:26.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_50 '..*' 2026-03-24T02:43:26.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:26.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:26.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_50 2026-03-24T02:43:26.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:43:26.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-03-24T02:43:26.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_50.status 2026-03-24T02:43:26.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:26.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:26.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-03-24T02:43:26.770 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:26.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:26.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:26.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:26.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:26.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2], [object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0]]], state=connected] 2026-03-24T02:43:26.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:26.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:26.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:26.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 'object_number=7, tag_tid=2, entry_tid=3' '!=' 'object_number=7, tag_tid=2, entry_tid=3' 2026-03-24T02:43:26.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-24T02:43:26.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.4 2026-03-24T02:43:27.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ flush cluster1 mirror image_50 2026-03-24T02:43:27.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:27.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:27.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:43:27.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local 'cmd=rbd mirror flush' 2026-03-24T02:43:27.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n image_50 ']' 2026-03-24T02:43:27.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmd='rbd mirror flush mirror/image_50' 2026-03-24T02:43:27.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_50 2026-03-24T02:43:27.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_instance=cluster1 2026-03-24T02:43:27.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:27.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:27.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance=cluster1 2026-03-24T02:43:27.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loop_instance 2026-03-24T02:43:27.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-24T02:43:27.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:27.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-24T02:43:27.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_50 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1:0 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local local_cluster=cluster1:0 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local val=cluster1:0 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster_var_name=local_cluster 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local instance_var_name=instance 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval local_cluster=cluster1 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ local_cluster=cluster1 2026-03-24T02:43:27.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ eval instance=0 2026-03-24T02:43:27.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ instance=0 2026-03-24T02:43:27.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-24T02:43:27.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:27.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:27.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-24T02:43:27.227 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_50 2026-03-24T02:43:27.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:27.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_master_journal_position cluster2 mirror image_50 2026-03-24T02:43:27.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:27.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:27.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_50 2026-03-24T02:43:27.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_50 '' 2026-03-24T02:43:27.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:27.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:27.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_50 2026-03-24T02:43:27.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local id_regexp= 2026-03-24T02:43:27.315 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-03-24T02:43:27.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_50.status 2026-03-24T02:43:27.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:27.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:27.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-03-24T02:43:27.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:27.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:27.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:27.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:27.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:27.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:27.357 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:27.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:27.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_mirror_journal_position cluster2 mirror image_50 2026-03-24T02:43:27.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:27.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:27.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_50 2026-03-24T02:43:27.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_journal_position cluster2 mirror image_50 '..*' 2026-03-24T02:43:27.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:43:27.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:43:27.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_50 2026-03-24T02:43:27.359 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local 'id_regexp=..*' 2026-03-24T02:43:27.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-03-24T02:43:27.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ echo cluster2-mirror-image_50.status 2026-03-24T02:43:27.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+++ sed -e 's|[/ ]|_|g' 2026-03-24T02:43:27.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local status_log=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:27.361 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-03-24T02:43:27.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ tee /tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:minimum_set: 1 2026-03-24T02:43:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:active_set: 1 2026-03-24T02:43:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:registered clients: 2026-03-24T02:43:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:27.401 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: [id=698fa6ae-5ebb-4bf6-b394-254b336fa2bd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2026-03-24T02:43:27.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.status 2026-03-24T02:43:27.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:27.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2026-03-24T02:43:27.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:27.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_snap_present cluster1 mirror image_50 snap 2026-03-24T02:43:27.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:43:27.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:27.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:43:27.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:43:27.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:43:27.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-24T02:43:27.407 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:43:28.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 info mirror/image_50@snap 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_50': 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 128 MiB in 32 objects 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 1 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 10d1d89f3e89 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.10d1d89f3e89 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:33:34 2026 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:33:34 2026 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:38:28 2026 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: protected: False 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 10d1d89f3e89 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 3e4758f9-72b3-45e8-89f2-5cb63feb5739 2026-03-24T02:43:28.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: false 2026-03-24T02:43:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ compare_image_snaps mirror image_50 snap 2026-03-24T02:43:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:43:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap_name=snap 2026-03-24T02:43:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local ret=0 2026-03-24T02:43:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local rmt_export=/tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.export 2026-03-24T02:43:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local loc_export=/tmp/tmp.l3GEA596QR/cluster1-mirror-image_50.export 2026-03-24T02:43:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_50.export 2026-03-24T02:43:28.454 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 -p mirror export image_50@snap /tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.export 2026-03-24T02:43:28.664 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:43:28.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster1 -p mirror export image_50@snap /tmp/tmp.l3GEA596QR/cluster1-mirror-image_50.export 2026-03-24T02:43:28.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-24T02:43:28.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cmp /tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_50.export 2026-03-24T02:43:28.990 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -f /tmp/tmp.l3GEA596QR/cluster2-mirror-image_50.export /tmp/tmp.l3GEA596QR/cluster1-mirror-image_50.export 2026-03-24T02:43:29.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:29.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ testlog 'TEST: delete many images' 2026-03-24T02:43:29.037 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ tee -a /tmp/tmp.l3GEA596QR/rbd-mirror.test.log 2026-03-24T02:43:29.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ date '+%F %T' 2026-03-24T02:43:29.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo 2026-03-24 02:43:29 TEST: delete many images 2026-03-24T02:43:29.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:2026-03-24 02:43:29 TEST: delete many images 2026-03-24T02:43:29.039 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 1 50 2026-03-24T02:43:29.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:29.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_1 2026-03-24T02:43:29.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_1 snap 2026-03-24T02:43:29.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:29.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:29.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:43:29.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:29.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_1@snap 2026-03-24T02:43:30.424 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:30.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_1 2026-03-24T02:43:30.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:30.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:30.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:43:30.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:30.438 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:30.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_1 2026-03-24T02:43:30.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:30.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:30.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:43:30.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-03-24T02:43:30.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-03-24T02:43:30.738 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:30.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:30.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:30.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_2 2026-03-24T02:43:30.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_2 snap 2026-03-24T02:43:30.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:30.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:30.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:43:30.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:30.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_2@snap 2026-03-24T02:43:31.103 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:31.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_2 2026-03-24T02:43:31.122 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:31.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:31.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:43:31.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:31.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:31.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_2 2026-03-24T02:43:31.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:31.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:31.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:43:31.123 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-03-24T02:43:31.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-03-24T02:43:31.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:31.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:31.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:31.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_3 2026-03-24T02:43:31.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_3 snap 2026-03-24T02:43:31.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:31.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:31.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:43:31.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:31.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_3@snap 2026-03-24T02:43:32.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:32.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_3 2026-03-24T02:43:32.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:32.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:32.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:43:32.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:32.195 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:32.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_3 2026-03-24T02:43:32.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:32.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:32.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:43:32.196 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-03-24T02:43:33.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-03-24T02:43:33.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-03-24T02:43:33.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:33.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:33.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:33.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_4 2026-03-24T02:43:33.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_4 snap 2026-03-24T02:43:33.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:33.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:33.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:43:33.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:33.505 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_4@snap 2026-03-24T02:43:34.188 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:34.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_4 2026-03-24T02:43:34.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:34.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:34.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:43:34.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:34.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:34.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_4 2026-03-24T02:43:34.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:34.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:34.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:43:34.206 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-03-24T02:43:34.273 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-03-24T02:43:34.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:34.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:34.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:34.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_5 2026-03-24T02:43:34.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_5 snap 2026-03-24T02:43:34.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:34.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:34.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:43:34.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:34.543 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_5@snap 2026-03-24T02:43:35.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:35.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_5 2026-03-24T02:43:35.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:35.193 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:35.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:43:35.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:35.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:35.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_5 2026-03-24T02:43:35.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:35.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:35.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:43:35.194 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-03-24T02:43:35.239 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-03-24T02:43:35.483 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:35.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:35.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:35.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_6 2026-03-24T02:43:35.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_6 snap 2026-03-24T02:43:35.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:35.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:35.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:43:35.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:35.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_6@snap 2026-03-24T02:43:36.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:36.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_6 2026-03-24T02:43:36.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:36.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:36.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:43:36.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:36.348 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:36.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_6 2026-03-24T02:43:36.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:36.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:36.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:43:36.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-03-24T02:43:36.406 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-03-24T02:43:37.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:37.203 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:37.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:37.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_7 2026-03-24T02:43:37.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_7 snap 2026-03-24T02:43:37.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:37.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:37.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:43:37.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:37.204 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_7@snap 2026-03-24T02:43:38.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:38.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_7 2026-03-24T02:43:38.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:38.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:38.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:43:38.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:38.344 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:38.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_7 2026-03-24T02:43:38.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:38.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:38.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:43:38.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-03-24T02:43:38.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-03-24T02:43:38.673 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9311045983996988) > weight (0.25) 2026-03-24T02:43:38.673 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-03-24T02:43:38.674 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-03-24T02:43:38.674 INFO:tasks.rbd_mirror_thrash:waiting for 4.709762482453755 secs before reviving daemons 2026-03-24T02:43:38.674 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:43:38.665+0000 7f56060d7640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 47823) UID: 1000 2026-03-24T02:43:38.726 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:38.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:38.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:38.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_8 2026-03-24T02:43:38.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_8 snap 2026-03-24T02:43:38.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:38.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:38.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:43:38.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:38.732 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_8@snap 2026-03-24T02:43:39.353 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:39.367 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_8 2026-03-24T02:43:39.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:39.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:39.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:43:39.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:39.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:39.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_8 2026-03-24T02:43:39.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:39.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:39.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:43:39.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-03-24T02:43:39.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-03-24T02:43:39.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:39.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:39.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:39.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_9 2026-03-24T02:43:39.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_9 snap 2026-03-24T02:43:39.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:39.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:39.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:43:39.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:39.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_9@snap 2026-03-24T02:43:40.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:40.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_9 2026-03-24T02:43:40.374 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:40.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:40.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:43:40.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:40.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:40.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_9 2026-03-24T02:43:40.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:40.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:40.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:43:40.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-03-24T02:43:40.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-03-24T02:43:40.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:40.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:40.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:40.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_10 2026-03-24T02:43:40.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_10 snap 2026-03-24T02:43:40.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:40.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:40.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:43:40.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:40.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_10@snap 2026-03-24T02:43:41.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:41.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_10 2026-03-24T02:43:41.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:41.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:41.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:43:41.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:41.372 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:41.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_10 2026-03-24T02:43:41.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:41.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:41.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:43:41.373 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-03-24T02:43:41.416 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-03-24T02:43:41.485 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:43:41.477+0000 7fe62256b640 -1 rbd::mirror::ImageMap: 0x55cd3d3de360 handle_peer_ack_remove: failed to remove global_image_id=af1b9448-880e-41c9-b13d-fd3d514c9e21 2026-03-24T02:43:41.677 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:41.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:41.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:41.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_11 2026-03-24T02:43:41.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_11 snap 2026-03-24T02:43:41.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:41.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:41.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:43:41.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:41.682 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_11@snap 2026-03-24T02:43:42.230 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:42.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_11 2026-03-24T02:43:42.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:42.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:42.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:43:42.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:42.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:42.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_11 2026-03-24T02:43:42.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:42.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:42.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:43:42.253 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_11 2026-03-24T02:43:42.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_11 2026-03-24T02:43:42.388 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:43:42.381+0000 7fe621d6a640 -1 rbd::mirror::ImageMap: 0x55cd3d3de360 handle_peer_ack_remove: failed to remove global_image_id=a029d6f0-f7cb-4e39-adc6-e3ec45e10c84 2026-03-24T02:43:42.573 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:42.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:42.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:42.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_12 2026-03-24T02:43:42.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_12 snap 2026-03-24T02:43:42.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:42.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:42.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:43:42.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:42.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_12@snap 2026-03-24T02:43:43.363 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:43.384 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-03-24T02:43:43.384 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:43:43.384 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-03-24T02:43:43.384 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-03-24T02:43:43.384 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-03-24T02:43:43.384 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_12 2026-03-24T02:43:43.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:43.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:43.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:43:43.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:43.385 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:43.386 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-03-24T02:43:43.386 INFO:tasks.rbd_mirror_thrash:waiting for 61.14482225332454 secs before thrashing 2026-03-24T02:43:43.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_12 2026-03-24T02:43:43.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:43.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:43.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:43:43.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_12 2026-03-24T02:43:43.427 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_12 2026-03-24T02:43:43.504 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:43:43.497+0000 7fe621d6a640 -1 rbd::mirror::ImageMap: 0x55cd3d3de360 handle_peer_ack_remove: failed to remove global_image_id=1ceb4471-c130-4ca5-8f54-c56dc41c52d6 2026-03-24T02:43:43.716 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:43.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:43.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:43.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_13 2026-03-24T02:43:43.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_13 snap 2026-03-24T02:43:43.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:43.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:43.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:43:43.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:43.721 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_13@snap 2026-03-24T02:43:44.368 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:44.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_13 2026-03-24T02:43:44.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:44.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:44.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:43:44.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:44.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:44.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_13 2026-03-24T02:43:44.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:44.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:44.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:43:44.388 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_13 2026-03-24T02:43:44.430 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_13 2026-03-24T02:43:44.479 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:43:44.473+0000 7fe621d6a640 -1 rbd::mirror::ImageMap: 0x55cd3d3de360 handle_peer_ack_remove: failed to remove global_image_id=fed73411-e3f3-4bc6-af5b-ff6461688863 2026-03-24T02:43:44.624 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:44.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:44.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:44.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_14 2026-03-24T02:43:44.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_14 snap 2026-03-24T02:43:44.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:44.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:44.629 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:43:44.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:44.630 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_14@snap 2026-03-24T02:43:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:45.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_14 2026-03-24T02:43:45.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:45.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:45.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:43:45.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:45.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:45.555 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_14 2026-03-24T02:43:45.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:45.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:45.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:43:45.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_14 2026-03-24T02:43:45.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_14 2026-03-24T02:43:46.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_15 2026-03-24T02:43:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_15 snap 2026-03-24T02:43:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:43:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_15@snap 2026-03-24T02:43:46.389 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:46.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_15 2026-03-24T02:43:46.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:46.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:46.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:43:46.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:46.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:46.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_15 2026-03-24T02:43:46.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:46.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:46.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:43:46.405 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_15 2026-03-24T02:43:46.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_15 2026-03-24T02:43:46.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:46.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:46.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:46.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_16 2026-03-24T02:43:46.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_16 snap 2026-03-24T02:43:46.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:46.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:46.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:43:46.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:46.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_16@snap 2026-03-24T02:43:47.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:47.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_16 2026-03-24T02:43:47.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:47.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:47.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:43:47.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:47.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:47.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_16 2026-03-24T02:43:47.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:47.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:47.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:43:47.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_16 2026-03-24T02:43:47.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_16 2026-03-24T02:43:47.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_17 2026-03-24T02:43:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_17 snap 2026-03-24T02:43:47.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:47.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:47.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:43:47.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:47.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_17@snap 2026-03-24T02:43:48.495 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:48.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_17 2026-03-24T02:43:48.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:48.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:48.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:43:48.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:48.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:48.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_17 2026-03-24T02:43:48.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:48.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:48.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:43:48.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_17 2026-03-24T02:43:48.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_17 2026-03-24T02:43:48.821 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:48.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:48.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:48.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_18 2026-03-24T02:43:48.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_18 snap 2026-03-24T02:43:48.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:48.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:48.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:43:48.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:48.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_18@snap 2026-03-24T02:43:49.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:49.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_18 2026-03-24T02:43:49.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:49.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:49.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:43:49.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:49.522 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:49.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_18 2026-03-24T02:43:49.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:49.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:49.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:43:49.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_18 2026-03-24T02:43:49.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_18 2026-03-24T02:43:49.804 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:49.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:49.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:49.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_19 2026-03-24T02:43:49.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_19 snap 2026-03-24T02:43:49.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:49.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:49.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:43:49.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:49.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_19@snap 2026-03-24T02:43:50.508 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:50.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_19 2026-03-24T02:43:50.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:50.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:50.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:43:50.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:50.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:50.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_19 2026-03-24T02:43:50.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:50.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:50.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:43:50.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_19 2026-03-24T02:43:50.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_19 2026-03-24T02:43:50.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:50.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:50.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:50.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_20 2026-03-24T02:43:50.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_20 snap 2026-03-24T02:43:50.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:50.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:50.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:43:50.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:50.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_20@snap 2026-03-24T02:43:51.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:51.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_20 2026-03-24T02:43:51.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:51.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:51.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:43:51.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:51.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:51.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_20 2026-03-24T02:43:51.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:51.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:51.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:43:51.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_20 2026-03-24T02:43:51.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_20 2026-03-24T02:43:51.623 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:43:51.617+0000 7fe62256b640 -1 rbd::mirror::ImageMap: 0x55cd3d3de360 handle_peer_ack_remove: failed to remove global_image_id=fb0f0a86-fc1a-4ffd-844c-8b7ff45aefbc 2026-03-24T02:43:51.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:51.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:51.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:51.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_21 2026-03-24T02:43:51.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_21 snap 2026-03-24T02:43:51.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:51.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:51.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:43:51.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:51.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_21@snap 2026-03-24T02:43:52.489 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:52.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_21 2026-03-24T02:43:52.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:52.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:52.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:43:52.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:52.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:52.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_21 2026-03-24T02:43:52.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:52.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:52.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:43:52.501 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_21 2026-03-24T02:43:52.544 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_21 2026-03-24T02:43:52.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_22 2026-03-24T02:43:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_22 snap 2026-03-24T02:43:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:43:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:52.806 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_22@snap 2026-03-24T02:43:53.509 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:53.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_22 2026-03-24T02:43:53.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:53.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:53.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:43:53.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:53.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:53.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_22 2026-03-24T02:43:53.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:53.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:53.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:43:53.521 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_22 2026-03-24T02:43:53.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_22 2026-03-24T02:43:53.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:53.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:53.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:53.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_23 2026-03-24T02:43:53.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_23 snap 2026-03-24T02:43:53.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:53.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:53.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:43:53.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:53.809 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_23@snap 2026-03-24T02:43:54.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:54.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_23 2026-03-24T02:43:54.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:54.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:54.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:43:54.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:54.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:54.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_23 2026-03-24T02:43:54.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:54.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:54.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:43:54.534 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_23 2026-03-24T02:43:54.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_23 2026-03-24T02:43:54.885 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:54.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:54.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:54.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_24 2026-03-24T02:43:54.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_24 snap 2026-03-24T02:43:54.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:54.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:54.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:43:54.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:54.890 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_24@snap 2026-03-24T02:43:55.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:55.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_24 2026-03-24T02:43:55.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:55.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:55.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:43:55.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:55.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:55.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_24 2026-03-24T02:43:55.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:55.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:55.530 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:43:55.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_24 2026-03-24T02:43:55.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_24 2026-03-24T02:43:55.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_25 2026-03-24T02:43:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_25 snap 2026-03-24T02:43:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:43:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_25@snap 2026-03-24T02:43:56.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:56.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_25 2026-03-24T02:43:56.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:56.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:56.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:43:56.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:56.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:56.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_25 2026-03-24T02:43:56.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:56.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:56.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:43:56.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_25 2026-03-24T02:43:56.592 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_25 2026-03-24T02:43:56.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:56.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:56.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:56.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_26 2026-03-24T02:43:56.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_26 snap 2026-03-24T02:43:56.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:56.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:56.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:43:56.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:56.828 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_26@snap 2026-03-24T02:43:57.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:57.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_26 2026-03-24T02:43:57.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:57.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:57.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:43:57.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:57.558 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:57.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_26 2026-03-24T02:43:57.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:57.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:57.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:43:57.559 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_26 2026-03-24T02:43:57.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_26 2026-03-24T02:43:57.934 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:57.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:57.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:57.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_27 2026-03-24T02:43:57.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_27 snap 2026-03-24T02:43:57.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:57.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:57.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:43:57.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:57.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_27@snap 2026-03-24T02:43:58.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_27 2026-03-24T02:43:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:43:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:58.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_27 2026-03-24T02:43:58.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:58.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:58.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:43:58.561 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_27 2026-03-24T02:43:58.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_27 2026-03-24T02:43:58.889 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:58.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:58.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:58.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_28 2026-03-24T02:43:58.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_28 snap 2026-03-24T02:43:58.893 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:58.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:58.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:43:58.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:58.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_28@snap 2026-03-24T02:43:59.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:43:59.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_28 2026-03-24T02:43:59.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:59.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:59.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:43:59.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:43:59.569 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:43:59.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_28 2026-03-24T02:43:59.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:59.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:59.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:43:59.570 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_28 2026-03-24T02:43:59.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_28 2026-03-24T02:43:59.865 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:43:59.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:43:59.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:43:59.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_29 2026-03-24T02:43:59.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_29 snap 2026-03-24T02:43:59.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:43:59.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:43:59.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:43:59.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:43:59.870 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_29@snap 2026-03-24T02:44:00.549 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:00.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_29 2026-03-24T02:44:00.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:00.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:00.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:44:00.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:00.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:00.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_29 2026-03-24T02:44:00.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:00.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:00.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:44:00.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_29 2026-03-24T02:44:00.605 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_29 2026-03-24T02:44:00.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:00.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:00.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:00.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_30 2026-03-24T02:44:00.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_30 snap 2026-03-24T02:44:00.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:00.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:00.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:44:00.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:00.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_30@snap 2026-03-24T02:44:01.565 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:01.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_30 2026-03-24T02:44:01.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:01.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:01.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:44:01.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:01.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:01.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_30 2026-03-24T02:44:01.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:01.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:01.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:44:01.579 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_30 2026-03-24T02:44:01.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_30 2026-03-24T02:44:01.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:01.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:01.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:01.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_31 2026-03-24T02:44:01.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_31 snap 2026-03-24T02:44:01.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:01.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:01.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:44:01.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:01.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_31@snap 2026-03-24T02:44:02.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:02.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_31 2026-03-24T02:44:02.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:02.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:02.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:44:02.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:02.517 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:02.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_31 2026-03-24T02:44:02.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:02.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:02.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:44:02.518 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_31 2026-03-24T02:44:02.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_31 2026-03-24T02:44:03.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:03.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:03.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:03.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_32 2026-03-24T02:44:03.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_32 snap 2026-03-24T02:44:03.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:03.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:03.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:44:03.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:03.018 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_32@snap 2026-03-24T02:44:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:03.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_32 2026-03-24T02:44:03.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:03.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:03.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:44:03.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:03.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:03.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_32 2026-03-24T02:44:03.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:03.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:03.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:44:03.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_32 2026-03-24T02:44:03.621 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_32 2026-03-24T02:44:03.682 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:44:03.673+0000 7fe62256b640 -1 rbd::mirror::ImageMap: 0x55cd3d3de360 handle_peer_ack_remove: failed to remove global_image_id=d4d47d9c-e0e4-4f91-b35c-a45a2ad4bfe9 2026-03-24T02:44:04.076 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:04.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:04.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:04.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_33 2026-03-24T02:44:04.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_33 snap 2026-03-24T02:44:04.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:04.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:04.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_33 2026-03-24T02:44:04.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:04.081 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_33@snap 2026-03-24T02:44:04.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:04.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_33 2026-03-24T02:44:04.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:04.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:04.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_33 2026-03-24T02:44:04.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:04.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:04.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_33 2026-03-24T02:44:04.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:04.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:04.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_33 2026-03-24T02:44:04.586 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_33 2026-03-24T02:44:04.625 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_33 2026-03-24T02:44:04.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:04.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:04.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:04.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_34 2026-03-24T02:44:04.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_34 snap 2026-03-24T02:44:04.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:04.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:04.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:44:04.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:04.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_34@snap 2026-03-24T02:44:05.571 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:05.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_34 2026-03-24T02:44:05.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:05.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:05.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:44:05.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:05.584 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:05.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_34 2026-03-24T02:44:05.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:05.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:05.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:44:05.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_34 2026-03-24T02:44:05.618 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_34 2026-03-24T02:44:05.850 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:05.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:05.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:05.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_35 2026-03-24T02:44:05.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_35 snap 2026-03-24T02:44:05.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:05.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:05.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:44:05.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:05.855 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_35@snap 2026-03-24T02:44:06.585 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:06.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_35 2026-03-24T02:44:06.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:06.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:06.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:44:06.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:06.603 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:06.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_35 2026-03-24T02:44:06.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:06.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:06.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:44:06.604 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_35 2026-03-24T02:44:06.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_35 2026-03-24T02:44:06.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:06.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:06.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:06.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_36 2026-03-24T02:44:06.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_36 snap 2026-03-24T02:44:06.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:06.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:06.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_36 2026-03-24T02:44:06.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:06.952 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_36@snap 2026-03-24T02:44:07.637 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:07.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_36 2026-03-24T02:44:07.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:07.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:07.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_36 2026-03-24T02:44:07.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:07.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:07.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_36 2026-03-24T02:44:07.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:07.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:07.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_36 2026-03-24T02:44:07.654 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_36 2026-03-24T02:44:07.703 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_36 2026-03-24T02:44:07.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:07.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:07.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:07.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_37 2026-03-24T02:44:07.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_37 snap 2026-03-24T02:44:07.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:07.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:07.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:44:07.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:07.964 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_37@snap 2026-03-24T02:44:08.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:08.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_37 2026-03-24T02:44:08.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:08.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:08.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:44:08.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:08.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:08.643 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_37 2026-03-24T02:44:08.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:08.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:08.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:44:08.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_37 2026-03-24T02:44:08.694 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_37 2026-03-24T02:44:08.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:08.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:08.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:08.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_38 2026-03-24T02:44:08.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_38 snap 2026-03-24T02:44:08.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:08.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:08.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:44:08.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:08.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_38@snap 2026-03-24T02:44:09.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:09.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_38 2026-03-24T02:44:09.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:09.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:09.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:44:09.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:09.655 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:09.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_38 2026-03-24T02:44:09.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:09.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:09.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:44:09.656 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_38 2026-03-24T02:44:09.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_38 2026-03-24T02:44:10.134 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_39 2026-03-24T02:44:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_39 snap 2026-03-24T02:44:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:44:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_39@snap 2026-03-24T02:44:10.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:10.678 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_39 2026-03-24T02:44:10.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:10.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:10.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:44:10.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:10.679 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:10.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_39 2026-03-24T02:44:10.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:10.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:10.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:44:10.680 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_39 2026-03-24T02:44:10.728 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_39 2026-03-24T02:44:10.974 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:10.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:10.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:10.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_40 2026-03-24T02:44:10.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_40 snap 2026-03-24T02:44:10.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:10.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:10.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:44:10.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:10.979 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_40@snap 2026-03-24T02:44:11.660 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:11.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_40 2026-03-24T02:44:11.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:11.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:11.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:44:11.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:11.673 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:11.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_40 2026-03-24T02:44:11.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:11.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:11.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:44:11.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_40 2026-03-24T02:44:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_40 2026-03-24T02:44:12.057 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_41 2026-03-24T02:44:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_41 snap 2026-03-24T02:44:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:44:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_41@snap 2026-03-24T02:44:12.504 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:12.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_41 2026-03-24T02:44:12.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:12.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:12.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:44:12.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:12.523 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:12.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_41 2026-03-24T02:44:12.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:12.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:12.524 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:44:12.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_41 2026-03-24T02:44:12.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_41 2026-03-24T02:44:13.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:13.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:13.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:13.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_42 2026-03-24T02:44:13.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_42 snap 2026-03-24T02:44:13.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:13.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:13.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:44:13.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:13.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_42@snap 2026-03-24T02:44:13.822 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:13.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_42 2026-03-24T02:44:13.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:13.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:13.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:44:13.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:13.836 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:13.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_42 2026-03-24T02:44:13.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:13.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:13.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:44:13.837 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_42 2026-03-24T02:44:13.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_42 2026-03-24T02:44:14.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:14.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:14.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:14.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_43 2026-03-24T02:44:14.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_43 snap 2026-03-24T02:44:14.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:14.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:14.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:44:14.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:14.173 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_43@snap 2026-03-24T02:44:14.813 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:14.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_43 2026-03-24T02:44:14.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:14.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:14.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:44:14.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:14.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_43 2026-03-24T02:44:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:44:14.825 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_43 2026-03-24T02:44:14.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_43 2026-03-24T02:44:15.089 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:15.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:15.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:15.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_44 2026-03-24T02:44:15.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_44 snap 2026-03-24T02:44:15.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:15.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:15.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:44:15.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:15.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_44@snap 2026-03-24T02:44:15.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:15.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_44 2026-03-24T02:44:15.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:15.856 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:15.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:44:15.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:15.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:15.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_44 2026-03-24T02:44:15.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:15.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:15.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:44:15.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_44 2026-03-24T02:44:15.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_44 2026-03-24T02:44:16.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_45 2026-03-24T02:44:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_45 snap 2026-03-24T02:44:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_45 2026-03-24T02:44:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_45@snap 2026-03-24T02:44:16.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:16.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_45 2026-03-24T02:44:16.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:16.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:16.853 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_45 2026-03-24T02:44:16.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:16.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:16.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_45 2026-03-24T02:44:16.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:16.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:16.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_45 2026-03-24T02:44:16.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_45 2026-03-24T02:44:16.894 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_45 2026-03-24T02:44:17.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:17.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:17.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:17.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_46 2026-03-24T02:44:17.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_46 snap 2026-03-24T02:44:17.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:17.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:17.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:44:17.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:17.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_46@snap 2026-03-24T02:44:17.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:17.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_46 2026-03-24T02:44:17.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:17.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:17.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:44:17.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:17.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:17.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_46 2026-03-24T02:44:17.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:17.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:17.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:44:17.858 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_46 2026-03-24T02:44:17.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_46 2026-03-24T02:44:18.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:18.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:18.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:18.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_47 2026-03-24T02:44:18.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_47 snap 2026-03-24T02:44:18.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:18.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:18.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:44:18.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:18.141 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_47@snap 2026-03-24T02:44:18.467 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:44:18.462+0000 7f5496ce4640 -1 reset not still connected to 0x5652c68aaa90 2026-03-24T02:44:18.474 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:44:18.466+0000 7f5496ce4640 -1 reset not still connected to 0x5652c83ca0d0 2026-03-24T02:44:18.474 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:44:18.466+0000 7f5496ce4640 -1 reset not still connected to 0x5652c8b12680 2026-03-24T02:44:18.474 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:44:18.466+0000 7f5496ce4640 -1 reset not still connected to 0x5652c8b13d40 2026-03-24T02:44:18.474 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:44:18.466+0000 7f5496ce4640 -1 reset not still connected to 0x5652ca2d4410 2026-03-24T02:44:18.474 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:44:18.466+0000 7f5496ce4640 -1 reset not still connected to 0x5652d23d3a00 2026-03-24T02:44:18.474 INFO:tasks.ceph.cluster1.osd.2.vm06.stderr:2026-03-24T02:44:18.466+0000 7f5496ce4640 -1 reset not still connected to 0x5652d714e340 2026-03-24T02:44:18.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:18.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_47 2026-03-24T02:44:18.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:18.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:18.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:44:18.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:18.859 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:18.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_47 2026-03-24T02:44:18.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:18.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:18.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:44:18.860 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_47 2026-03-24T02:44:18.900 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_47 2026-03-24T02:44:19.137 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:19.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:19.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:19.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_48 2026-03-24T02:44:19.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_48 snap 2026-03-24T02:44:19.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:19.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:19.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_48 2026-03-24T02:44:19.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:19.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_48@snap 2026-03-24T02:44:19.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:19.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_48 2026-03-24T02:44:19.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:19.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:19.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_48 2026-03-24T02:44:19.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:19.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:19.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_48 2026-03-24T02:44:19.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:19.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:19.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_48 2026-03-24T02:44:19.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_48 2026-03-24T02:44:19.898 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_48 2026-03-24T02:44:20.151 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:20.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:20.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:20.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_49 2026-03-24T02:44:20.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_49 snap 2026-03-24T02:44:20.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:20.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:20.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:44:20.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:20.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_49@snap 2026-03-24T02:44:20.861 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:20.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_49 2026-03-24T02:44:20.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:20.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:20.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:44:20.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:20.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:20.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_49 2026-03-24T02:44:20.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:20.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:20.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:44:20.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_49 2026-03-24T02:44:20.905 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_49 2026-03-24T02:44:21.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:21.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:21.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:21.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_50 2026-03-24T02:44:21.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_snapshot cluster2 mirror image_50 snap 2026-03-24T02:44:21.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:21.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:21.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:44:21.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local snap=snap 2026-03-24T02:44:21.168 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap rm mirror/image_50@snap 2026-03-24T02:44:21.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing snap: 100% complete...done. 2026-03-24T02:44:21.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_50 2026-03-24T02:44:21.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:21.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:21.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:44:21.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:44:21.916 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:44:21.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_50 2026-03-24T02:44:21.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:21.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:21.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:44:21.917 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_50 2026-03-24T02:44:21.949 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_50 2026-03-24T02:44:22.201 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-24T02:44:22.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:22.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ testlog 'TEST: image deletions should propagate' 2026-03-24T02:44:22.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ tee -a /tmp/tmp.l3GEA596QR/rbd-mirror.test.log 2026-03-24T02:44:22.207 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ date '+%F %T' 2026-03-24T02:44:22.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo 2026-03-24 02:44:22 TEST: image deletions should propagate 2026-03-24T02:44:22.208 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:2026-03-24 02:44:22 TEST: image deletions should propagate 2026-03-24T02:44:22.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_pool_images cluster1 mirror 0 2026-03-24T02:44:22.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:22.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:22.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_count=0 2026-03-24T02:44:22.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:22.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local count 2026-03-24T02:44:22.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local last_count=0 2026-03-24T02:44:22.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ true 2026-03-24T02:44:22.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 1 40 2026-03-24T02:44:22.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in `seq 1 40` 2026-03-24T02:44:22.210 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 1 -ne 1 2026-03-24T02:44:22.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-03-24T02:44:22.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep 'images: ' 2026-03-24T02:44:22.211 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut '-d ' -f 2 2026-03-24T02:44:22.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: failed to query services: (13) Permission denied 2026-03-24T02:44:22.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ count=0 2026-03-24T02:44:22.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 0 = 0 2026-03-24T02:44:22.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:22.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_pool_healthy cluster1 mirror 0 2026-03-24T02:44:22.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:22.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:22.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:22.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state 2026-03-24T02:44:22.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 1 40 2026-03-24T02:44:22.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in `seq 1 40` 2026-03-24T02:44:22.242 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 1 -ne 1 2026-03-24T02:44:22.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-03-24T02:44:22.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep 'image health:' 2026-03-24T02:44:22.243 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cut '-d ' -f 3 2026-03-24T02:44:22.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: failed to query services: (13) Permission denied 2026-03-24T02:44:22.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ state=OK 2026-03-24T02:44:22.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test OK = ERROR 2026-03-24T02:44:22.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test OK = OK 2026-03-24T02:44:22.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:22.275 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 1 50 2026-03-24T02:44:22.276 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_1 2026-03-24T02:44:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2026-03-24T02:44:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:44:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_1 2026-03-24T02:44:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:44:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-03-24T02:44:22.277 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:22.303 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-03-24T02:44:22.308 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:22.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:22.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:22.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2026-03-24T02:44:22.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:22.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:22.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:44:22.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:22.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:22.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:22.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:22.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_1 2026-03-24T02:44:22.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:22.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:22.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:44:22.410 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-03-24T02:44:22.411 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:22.445 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_2 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_2 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:22.450 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:44:22.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-03-24T02:44:22.451 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:22.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-03-24T02:44:22.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:22.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:22.500 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:22.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2026-03-24T02:44:22.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:22.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:22.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:44:22.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:22.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:22.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:22.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:22.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_2 2026-03-24T02:44:22.601 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:22.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:22.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:44:22.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-03-24T02:44:22.602 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:22.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_3 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_3 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-03-24T02:44:22.631 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:22.659 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-03-24T02:44:22.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:22.662 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:22.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:22.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2026-03-24T02:44:22.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:22.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:22.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:44:22.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:22.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:22.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:22.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:22.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_3 2026-03-24T02:44:22.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:22.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:22.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:44:22.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-03-24T02:44:22.765 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:22.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-03-24T02:44:22.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:22.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:22.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:22.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:22.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:22.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_4 2026-03-24T02:44:22.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2026-03-24T02:44:22.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:22.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:22.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:44:22.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:22.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:22.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:22.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:22.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_4 2026-03-24T02:44:22.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:22.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:22.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_4 2026-03-24T02:44:22.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-03-24T02:44:22.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:22.835 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-03-24T02:44:22.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:22.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:22.840 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:22.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2026-03-24T02:44:22.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:22.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:22.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:44:22.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:22.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:22.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:22.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:22.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_4 2026-03-24T02:44:22.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:22.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:22.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_4 2026-03-24T02:44:22.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-03-24T02:44:22.942 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:22.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-03-24T02:44:22.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:22.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:22.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:22.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:22.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:22.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_5 2026-03-24T02:44:22.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2026-03-24T02:44:22.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:22.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:22.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:44:22.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:22.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:22.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:22.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:22.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_5 2026-03-24T02:44:22.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:22.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:22.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:44:22.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-03-24T02:44:22.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:23.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-03-24T02:44:23.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:23.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:23.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:23.106 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2026-03-24T02:44:23.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:23.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:23.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:44:23.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:23.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:23.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:23.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:23.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_5 2026-03-24T02:44:23.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:23.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:23.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:44:23.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-03-24T02:44:23.107 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:23.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-03-24T02:44:23.138 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_6 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_6 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-03-24T02:44:23.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:23.164 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-03-24T02:44:23.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:23.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:23.167 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2026-03-24T02:44:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:44:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:23.268 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_6 2026-03-24T02:44:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:44:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-03-24T02:44:23.269 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:23.296 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-03-24T02:44:23.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:23.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:23.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:23.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:23.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:23.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_7 2026-03-24T02:44:23.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2026-03-24T02:44:23.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:23.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:23.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:44:23.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:23.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:23.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:23.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:23.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_7 2026-03-24T02:44:23.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:23.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:23.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:44:23.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-03-24T02:44:23.302 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:23.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-03-24T02:44:23.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:23.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:23.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:23.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2026-03-24T02:44:23.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:23.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:23.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:44:23.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:23.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:23.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:23.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:23.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_7 2026-03-24T02:44:23.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:23.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:23.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:44:23.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-03-24T02:44:23.433 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:23.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-03-24T02:44:23.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:23.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:23.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:23.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:23.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:23.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_8 2026-03-24T02:44:23.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2026-03-24T02:44:23.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:23.470 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:23.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:44:23.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:23.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:23.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:23.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:23.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_8 2026-03-24T02:44:23.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:23.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:23.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:44:23.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-03-24T02:44:23.471 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:23.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-03-24T02:44:23.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:23.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:23.511 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:23.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2026-03-24T02:44:23.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:23.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:23.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:44:23.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:23.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:23.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:23.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:23.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_8 2026-03-24T02:44:23.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:23.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:23.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:44:23.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-03-24T02:44:23.613 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:23.644 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-03-24T02:44:23.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:23.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:23.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:23.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:23.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:23.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_9 2026-03-24T02:44:23.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2026-03-24T02:44:23.647 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:23.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:23.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:44:23.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:23.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:23.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:23.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:23.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_9 2026-03-24T02:44:23.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:23.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:23.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:44:23.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-03-24T02:44:23.648 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:23.672 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-03-24T02:44:23.674 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:23.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:23.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:23.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2026-03-24T02:44:23.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:23.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:23.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:44:23.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:23.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:23.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:23.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:23.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_9 2026-03-24T02:44:23.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:23.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:23.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:44:23.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-03-24T02:44:23.776 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:23.807 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-03-24T02:44:23.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:23.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:23.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:23.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:23.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:23.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_10 2026-03-24T02:44:23.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2026-03-24T02:44:23.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:23.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:23.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:44:23.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:23.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:23.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:23.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:23.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_10 2026-03-24T02:44:23.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:23.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:23.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_10 2026-03-24T02:44:23.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-03-24T02:44:23.812 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:23.838 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-03-24T02:44:23.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:23.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:23.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:23.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2026-03-24T02:44:23.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:23.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:23.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:44:23.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:23.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:23.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:23.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:23.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_10 2026-03-24T02:44:23.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:23.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:23.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_10 2026-03-24T02:44:23.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-03-24T02:44:23.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:23.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-03-24T02:44:23.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:23.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:23.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:23.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:23.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:23.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_11 2026-03-24T02:44:23.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_11 deleted 2026-03-24T02:44:23.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_11 2026-03-24T02:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2026-03-24T02:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:24.007 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_11: (2) No such file or directory 2026-03-24T02:44:24.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:24.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:24.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:24.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_11 deleted '' 2026-03-24T02:44:24.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:24.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:24.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_11 2026-03-24T02:44:24.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:24.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:24.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:24.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:24.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_11 2026-03-24T02:44:24.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:24.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:24.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_11 2026-03-24T02:44:24.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2026-03-24T02:44:24.115 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:24.142 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_11: (2) No such file or directory 2026-03-24T02:44:24.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:24.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_12 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_12 deleted 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_12 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2026-03-24T02:44:24.148 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:24.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_12: (2) No such file or directory 2026-03-24T02:44:24.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:24.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:24.182 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:24.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_12 deleted '' 2026-03-24T02:44:24.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:24.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:24.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_12 2026-03-24T02:44:24.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:24.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:24.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:24.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:24.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_12 2026-03-24T02:44:24.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:24.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:24.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_12 2026-03-24T02:44:24.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2026-03-24T02:44:24.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:24.309 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_12: (2) No such file or directory 2026-03-24T02:44:24.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:24.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:24.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:24.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:24.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:24.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_13 2026-03-24T02:44:24.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_13 deleted 2026-03-24T02:44:24.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:24.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:24.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:44:24.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:24.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:24.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:24.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:24.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_13 2026-03-24T02:44:24.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:24.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:24.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:44:24.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2026-03-24T02:44:24.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:24.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_13: (2) No such file or directory 2026-03-24T02:44:24.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:24.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:24.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:24.439 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_13 deleted '' 2026-03-24T02:44:24.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:24.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:24.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_13 2026-03-24T02:44:24.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:24.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:24.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:24.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:24.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_13 2026-03-24T02:44:24.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:24.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:24.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_13 2026-03-24T02:44:24.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2026-03-24T02:44:24.440 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:24.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_13: (2) No such file or directory 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_14 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_14 deleted 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_14 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:24.513 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:44:24.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2026-03-24T02:44:24.514 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:24.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_14: (2) No such file or directory 2026-03-24T02:44:24.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:24.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:24.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:24.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_14 deleted '' 2026-03-24T02:44:24.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:24.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:24.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_14 2026-03-24T02:44:24.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:24.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:24.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:24.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:24.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_14 2026-03-24T02:44:24.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:24.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:24.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_14 2026-03-24T02:44:24.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2026-03-24T02:44:24.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:24.681 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_14: (2) No such file or directory 2026-03-24T02:44:24.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:24.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:24.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:24.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:24.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:24.685 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_15 2026-03-24T02:44:24.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_15 deleted 2026-03-24T02:44:24.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:24.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:24.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:44:24.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:24.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:24.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:24.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:24.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_15 2026-03-24T02:44:24.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:24.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:24.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_15 2026-03-24T02:44:24.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2026-03-24T02:44:24.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:24.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_15: (2) No such file or directory 2026-03-24T02:44:24.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:24.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:24.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_15 deleted '' 2026-03-24T02:44:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_15 2026-03-24T02:44:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_15 2026-03-24T02:44:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_15 2026-03-24T02:44:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2026-03-24T02:44:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:24.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_15: (2) No such file or directory 2026-03-24T02:44:24.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:24.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:24.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:24.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:24.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:24.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_16 2026-03-24T02:44:24.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_16 deleted 2026-03-24T02:44:24.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:24.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:24.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:44:24.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:24.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:24.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:24.847 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:24.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_16 2026-03-24T02:44:24.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:24.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:24.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_16 2026-03-24T02:44:24.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2026-03-24T02:44:24.848 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:24.872 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_16: (2) No such file or directory 2026-03-24T02:44:24.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:24.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:24.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:24.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_16 deleted '' 2026-03-24T02:44:24.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:24.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:24.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_16 2026-03-24T02:44:24.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:24.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:24.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:24.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:24.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_16 2026-03-24T02:44:24.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:24.977 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:24.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_16 2026-03-24T02:44:24.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2026-03-24T02:44:24.978 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:25.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_16: (2) No such file or directory 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_17 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_17 deleted 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_17 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_17 2026-03-24T02:44:25.009 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2026-03-24T02:44:25.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:25.038 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_17: (2) No such file or directory 2026-03-24T02:44:25.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:25.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:25.043 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:25.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_17 deleted '' 2026-03-24T02:44:25.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:25.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:25.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_17 2026-03-24T02:44:25.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:25.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:25.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:25.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:25.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_17 2026-03-24T02:44:25.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:25.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:25.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_17 2026-03-24T02:44:25.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2026-03-24T02:44:25.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:25.171 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_17: (2) No such file or directory 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_18 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_18 deleted 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_18 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2026-03-24T02:44:25.205 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_18: (2) No such file or directory 2026-03-24T02:44:25.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:25.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:25.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:25.310 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_18 deleted '' 2026-03-24T02:44:25.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:25.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:25.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_18 2026-03-24T02:44:25.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:25.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:25.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:25.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:25.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_18 2026-03-24T02:44:25.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:25.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:25.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_18 2026-03-24T02:44:25.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2026-03-24T02:44:25.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:25.340 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_18: (2) No such file or directory 2026-03-24T02:44:25.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:25.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:25.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:25.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:25.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:25.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_19 2026-03-24T02:44:25.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_19 deleted 2026-03-24T02:44:25.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:25.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:25.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:44:25.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:25.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:25.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:25.345 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:25.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_19 2026-03-24T02:44:25.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:25.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:25.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:44:25.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:25.346 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2026-03-24T02:44:25.375 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_19: (2) No such file or directory 2026-03-24T02:44:25.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:25.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:25.380 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:25.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_19 deleted '' 2026-03-24T02:44:25.481 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:25.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:25.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_19 2026-03-24T02:44:25.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:25.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:25.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:25.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:25.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_19 2026-03-24T02:44:25.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:25.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:25.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_19 2026-03-24T02:44:25.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2026-03-24T02:44:25.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:25.531 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_19: (2) No such file or directory 2026-03-24T02:44:25.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:25.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:25.536 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:25.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:25.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:25.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_20 2026-03-24T02:44:25.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_20 deleted 2026-03-24T02:44:25.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:25.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:25.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:44:25.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:25.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:25.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:25.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:25.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_20 2026-03-24T02:44:25.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:25.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:25.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_20 2026-03-24T02:44:25.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2026-03-24T02:44:25.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:25.578 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_20: (2) No such file or directory 2026-03-24T02:44:25.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:25.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:25.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:25.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_20 deleted '' 2026-03-24T02:44:25.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:25.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:25.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_20 2026-03-24T02:44:25.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:25.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:25.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:25.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:25.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_20 2026-03-24T02:44:25.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:25.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:25.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_20 2026-03-24T02:44:25.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2026-03-24T02:44:25.684 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:25.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_20: (2) No such file or directory 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_21 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_21 deleted 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_21 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:44:25.719 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2026-03-24T02:44:25.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:25.746 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_21: (2) No such file or directory 2026-03-24T02:44:25.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:25.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:25.750 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_21 deleted '' 2026-03-24T02:44:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_21 2026-03-24T02:44:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_21 2026-03-24T02:44:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_21 2026-03-24T02:44:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2026-03-24T02:44:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:25.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_21: (2) No such file or directory 2026-03-24T02:44:25.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:25.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:25.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:25.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:25.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:25.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_22 2026-03-24T02:44:25.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_22 deleted 2026-03-24T02:44:25.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:25.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:25.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:44:25.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:25.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:25.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:25.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:25.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_22 2026-03-24T02:44:25.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:25.881 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:25.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:44:25.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2026-03-24T02:44:25.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:25.907 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_22: (2) No such file or directory 2026-03-24T02:44:25.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:25.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:25.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:26.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_22 deleted '' 2026-03-24T02:44:26.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:26.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:26.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_22 2026-03-24T02:44:26.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:26.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:26.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:26.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:26.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_22 2026-03-24T02:44:26.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:26.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:26.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_22 2026-03-24T02:44:26.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2026-03-24T02:44:26.013 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:26.040 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_22: (2) No such file or directory 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_23 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_23 deleted 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_23 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2026-03-24T02:44:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:26.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_23: (2) No such file or directory 2026-03-24T02:44:26.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:26.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:26.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:26.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_23 deleted '' 2026-03-24T02:44:26.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:26.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:26.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_23 2026-03-24T02:44:26.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:26.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:26.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:26.180 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:26.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_23 2026-03-24T02:44:26.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:26.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:26.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_23 2026-03-24T02:44:26.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2026-03-24T02:44:26.181 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:26.209 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_23: (2) No such file or directory 2026-03-24T02:44:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_24 2026-03-24T02:44:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_24 deleted 2026-03-24T02:44:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:44:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:26.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:26.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_24 2026-03-24T02:44:26.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:26.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:26.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_24 2026-03-24T02:44:26.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2026-03-24T02:44:26.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:26.240 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_24: (2) No such file or directory 2026-03-24T02:44:26.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:26.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:26.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:26.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_24 deleted '' 2026-03-24T02:44:26.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:26.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:26.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_24 2026-03-24T02:44:26.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:26.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:26.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:26.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:26.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_24 2026-03-24T02:44:26.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:26.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:26.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_24 2026-03-24T02:44:26.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2026-03-24T02:44:26.347 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:26.376 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_24: (2) No such file or directory 2026-03-24T02:44:26.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:26.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:26.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:26.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:26.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:26.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_25 2026-03-24T02:44:26.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_25 deleted 2026-03-24T02:44:26.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:26.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:26.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:44:26.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:26.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:26.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:26.381 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_25 2026-03-24T02:44:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:44:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2026-03-24T02:44:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:26.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_25: (2) No such file or directory 2026-03-24T02:44:26.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:26.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:26.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:26.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_25 deleted '' 2026-03-24T02:44:26.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:26.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:26.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_25 2026-03-24T02:44:26.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:26.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:26.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:26.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:26.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_25 2026-03-24T02:44:26.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:26.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:26.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_25 2026-03-24T02:44:26.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2026-03-24T02:44:26.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:26.583 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_25: (2) No such file or directory 2026-03-24T02:44:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_26 2026-03-24T02:44:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_26 deleted 2026-03-24T02:44:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:44:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:26.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_26 2026-03-24T02:44:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:44:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2026-03-24T02:44:26.589 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:26.617 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_26: (2) No such file or directory 2026-03-24T02:44:26.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:26.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:26.620 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_26 deleted '' 2026-03-24T02:44:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_26 2026-03-24T02:44:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_26 2026-03-24T02:44:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_26 2026-03-24T02:44:26.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2026-03-24T02:44:26.723 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:26.749 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_26: (2) No such file or directory 2026-03-24T02:44:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_27 2026-03-24T02:44:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_27 deleted 2026-03-24T02:44:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:44:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_27 2026-03-24T02:44:26.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:26.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:26.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:44:26.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2026-03-24T02:44:26.756 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:26.785 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_27: (2) No such file or directory 2026-03-24T02:44:26.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:26.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:26.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:26.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_27 deleted '' 2026-03-24T02:44:26.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:26.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:26.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_27 2026-03-24T02:44:26.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:26.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:26.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:26.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:26.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_27 2026-03-24T02:44:26.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:26.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:26.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_27 2026-03-24T02:44:26.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2026-03-24T02:44:26.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:26.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_27: (2) No such file or directory 2026-03-24T02:44:26.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:26.923 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_28 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_28 deleted 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_28 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2026-03-24T02:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:26.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_28: (2) No such file or directory 2026-03-24T02:44:26.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:26.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:26.954 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:27.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_28 deleted '' 2026-03-24T02:44:27.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:27.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:27.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_28 2026-03-24T02:44:27.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:27.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:27.055 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:27.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:27.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_28 2026-03-24T02:44:27.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:27.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:27.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_28 2026-03-24T02:44:27.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2026-03-24T02:44:27.056 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:27.084 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_28: (2) No such file or directory 2026-03-24T02:44:27.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:27.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:27.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:27.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:27.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:27.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_29 2026-03-24T02:44:27.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_29 deleted 2026-03-24T02:44:27.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:27.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:27.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:44:27.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:27.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:27.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:27.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:27.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_29 2026-03-24T02:44:27.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:27.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:27.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_29 2026-03-24T02:44:27.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2026-03-24T02:44:27.088 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:27.114 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_29: (2) No such file or directory 2026-03-24T02:44:27.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:27.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:27.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:27.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_29 deleted '' 2026-03-24T02:44:27.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:27.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:27.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_29 2026-03-24T02:44:27.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:27.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:27.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:27.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:27.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_29 2026-03-24T02:44:27.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:27.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:27.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_29 2026-03-24T02:44:27.219 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2026-03-24T02:44:27.220 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:27.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_29: (2) No such file or directory 2026-03-24T02:44:27.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:27.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:27.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:27.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:27.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:27.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_30 2026-03-24T02:44:27.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_30 deleted 2026-03-24T02:44:27.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:27.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:27.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:44:27.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:27.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:27.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:27.248 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:27.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_30 2026-03-24T02:44:27.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:27.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:27.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:44:27.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2026-03-24T02:44:27.249 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:27.280 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_30: (2) No such file or directory 2026-03-24T02:44:27.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:27.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:27.284 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:27.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_30 deleted '' 2026-03-24T02:44:27.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:27.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:27.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_30 2026-03-24T02:44:27.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:27.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:27.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:27.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:27.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_30 2026-03-24T02:44:27.386 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:27.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:27.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_30 2026-03-24T02:44:27.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2026-03-24T02:44:27.387 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:27.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_30: (2) No such file or directory 2026-03-24T02:44:27.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:27.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:27.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:27.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:27.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:27.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_31 2026-03-24T02:44:27.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_31 deleted 2026-03-24T02:44:27.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:27.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:27.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:44:27.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:27.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:27.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:27.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:27.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_31 2026-03-24T02:44:27.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:27.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:27.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:44:27.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2026-03-24T02:44:27.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:27.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_31: (2) No such file or directory 2026-03-24T02:44:27.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:27.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:27.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:27.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_31 deleted '' 2026-03-24T02:44:27.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:27.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:27.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_31 2026-03-24T02:44:27.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:27.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:27.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:27.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:27.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_31 2026-03-24T02:44:27.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:27.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:27.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_31 2026-03-24T02:44:27.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2026-03-24T02:44:27.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:27.591 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_31: (2) No such file or directory 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_32 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_32 deleted 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_32 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2026-03-24T02:44:27.597 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:27.636 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_32: (2) No such file or directory 2026-03-24T02:44:27.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:27.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:27.641 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:27.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_32 deleted '' 2026-03-24T02:44:27.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:27.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:27.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_32 2026-03-24T02:44:27.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:27.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:27.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:27.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:27.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_32 2026-03-24T02:44:27.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:27.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:27.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_32 2026-03-24T02:44:27.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2026-03-24T02:44:27.743 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:27.769 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_32: (2) No such file or directory 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_33 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_33 deleted 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_33 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_33 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_33 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2026-03-24T02:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:27.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_33: (2) No such file or directory 2026-03-24T02:44:27.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:27.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:27.808 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:27.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_33 deleted '' 2026-03-24T02:44:27.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:27.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:27.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_33 2026-03-24T02:44:27.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:27.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:27.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:27.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:27.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_33 2026-03-24T02:44:27.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:27.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:27.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_33 2026-03-24T02:44:27.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2026-03-24T02:44:27.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:27.936 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_33: (2) No such file or directory 2026-03-24T02:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_34 2026-03-24T02:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_34 deleted 2026-03-24T02:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_34 2026-03-24T02:44:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:44:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2026-03-24T02:44:27.941 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:27.968 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_34: (2) No such file or directory 2026-03-24T02:44:27.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:27.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:27.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:28.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_34 deleted '' 2026-03-24T02:44:28.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:28.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:28.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_34 2026-03-24T02:44:28.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:28.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:28.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:28.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:28.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_34 2026-03-24T02:44:28.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:28.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:28.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_34 2026-03-24T02:44:28.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2026-03-24T02:44:28.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:28.105 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_34: (2) No such file or directory 2026-03-24T02:44:28.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:28.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:28.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:28.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:28.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:28.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_35 2026-03-24T02:44:28.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_35 deleted 2026-03-24T02:44:28.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:28.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:28.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:44:28.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:28.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:28.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:28.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:28.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_35 2026-03-24T02:44:28.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:28.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:28.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:44:28.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2026-03-24T02:44:28.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:28.139 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_35: (2) No such file or directory 2026-03-24T02:44:28.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:28.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:28.143 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:28.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_35 deleted '' 2026-03-24T02:44:28.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:28.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:28.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_35 2026-03-24T02:44:28.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:28.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:28.244 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:28.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:28.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_35 2026-03-24T02:44:28.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:28.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:28.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_35 2026-03-24T02:44:28.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2026-03-24T02:44:28.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:28.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_35: (2) No such file or directory 2026-03-24T02:44:28.278 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_36 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_36 deleted 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_36 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_36 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_36 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2026-03-24T02:44:28.279 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:28.307 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_36: (2) No such file or directory 2026-03-24T02:44:28.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:28.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:28.312 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:28.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_36 deleted '' 2026-03-24T02:44:28.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:28.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:28.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_36 2026-03-24T02:44:28.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:28.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:28.413 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:28.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:28.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_36 2026-03-24T02:44:28.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:28.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:28.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_36 2026-03-24T02:44:28.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2026-03-24T02:44:28.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:28.446 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_36: (2) No such file or directory 2026-03-24T02:44:28.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:28.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:28.452 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_37 2026-03-24T02:44:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_37 deleted 2026-03-24T02:44:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:44:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_37 2026-03-24T02:44:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:44:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2026-03-24T02:44:28.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:28.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_37: (2) No such file or directory 2026-03-24T02:44:28.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:28.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:28.485 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:28.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_37 deleted '' 2026-03-24T02:44:28.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:28.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:28.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_37 2026-03-24T02:44:28.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:28.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:28.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:28.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:28.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_37 2026-03-24T02:44:28.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:28.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:28.587 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_37 2026-03-24T02:44:28.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2026-03-24T02:44:28.588 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:28.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_37: (2) No such file or directory 2026-03-24T02:44:28.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:28.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:28.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:28.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:28.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:28.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_38 2026-03-24T02:44:28.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_38 deleted 2026-03-24T02:44:28.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:28.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:28.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:44:28.633 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:28.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:28.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:28.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:28.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_38 2026-03-24T02:44:28.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:28.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:28.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:44:28.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2026-03-24T02:44:28.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:28.663 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_38: (2) No such file or directory 2026-03-24T02:44:28.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:28.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:28.670 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:28.771 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_38 deleted '' 2026-03-24T02:44:28.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:28.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:28.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_38 2026-03-24T02:44:28.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:28.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:28.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:28.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:28.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_38 2026-03-24T02:44:28.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:28.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:28.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_38 2026-03-24T02:44:28.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2026-03-24T02:44:28.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:28.798 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_38: (2) No such file or directory 2026-03-24T02:44:28.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:28.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:28.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:28.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:28.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:28.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_39 2026-03-24T02:44:28.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_39 deleted 2026-03-24T02:44:28.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:28.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:28.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:44:28.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:28.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:28.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:28.802 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:28.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_39 2026-03-24T02:44:28.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:28.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:28.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_39 2026-03-24T02:44:28.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2026-03-24T02:44:28.803 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:28.826 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_39: (2) No such file or directory 2026-03-24T02:44:28.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:28.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:28.829 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:28.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_39 deleted '' 2026-03-24T02:44:28.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:28.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:28.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_39 2026-03-24T02:44:28.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:28.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:28.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:28.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:28.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_39 2026-03-24T02:44:28.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:28.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:28.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_39 2026-03-24T02:44:28.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2026-03-24T02:44:28.931 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:28.958 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_39: (2) No such file or directory 2026-03-24T02:44:28.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:28.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:28.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:28.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:28.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:28.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_40 2026-03-24T02:44:28.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_40 deleted 2026-03-24T02:44:28.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:28.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:28.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:44:28.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:28.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:28.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:28.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:28.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_40 2026-03-24T02:44:28.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:28.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:28.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_40 2026-03-24T02:44:28.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2026-03-24T02:44:28.963 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:28.988 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_40: (2) No such file or directory 2026-03-24T02:44:28.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:28.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:28.992 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:29.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_40 deleted '' 2026-03-24T02:44:29.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:29.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:29.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_40 2026-03-24T02:44:29.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:29.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:29.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:29.093 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:29.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_40 2026-03-24T02:44:29.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:29.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:29.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_40 2026-03-24T02:44:29.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2026-03-24T02:44:29.094 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:29.121 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_40: (2) No such file or directory 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_41 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_41 deleted 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_41 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_41 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:29.125 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2026-03-24T02:44:29.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_41: (2) No such file or directory 2026-03-24T02:44:29.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:29.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:29.161 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:29.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_41 deleted '' 2026-03-24T02:44:29.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:29.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:29.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_41 2026-03-24T02:44:29.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:29.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:29.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:29.262 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:29.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_41 2026-03-24T02:44:29.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:29.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:29.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_41 2026-03-24T02:44:29.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2026-03-24T02:44:29.263 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:29.287 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_41: (2) No such file or directory 2026-03-24T02:44:29.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:29.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:29.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:29.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:29.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:29.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_42 2026-03-24T02:44:29.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_42 deleted 2026-03-24T02:44:29.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:29.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:29.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:44:29.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:29.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:29.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:29.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:29.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_42 2026-03-24T02:44:29.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:29.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:29.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:44:29.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2026-03-24T02:44:29.291 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:29.313 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_42: (2) No such file or directory 2026-03-24T02:44:29.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:29.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:29.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:29.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_42 deleted '' 2026-03-24T02:44:29.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:29.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:29.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_42 2026-03-24T02:44:29.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:29.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:29.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:29.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:29.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_42 2026-03-24T02:44:29.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:29.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:29.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_42 2026-03-24T02:44:29.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2026-03-24T02:44:29.419 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:29.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_42: (2) No such file or directory 2026-03-24T02:44:29.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:29.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:29.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:29.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:29.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:29.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_43 2026-03-24T02:44:29.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_43 deleted 2026-03-24T02:44:29.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:29.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:29.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:44:29.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:29.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:29.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:29.447 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:29.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_43 2026-03-24T02:44:29.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:29.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:29.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:44:29.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2026-03-24T02:44:29.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:29.476 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_43: (2) No such file or directory 2026-03-24T02:44:29.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:29.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:29.480 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:29.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_43 deleted '' 2026-03-24T02:44:29.581 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:29.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:29.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_43 2026-03-24T02:44:29.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:29.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:29.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:29.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:29.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_43 2026-03-24T02:44:29.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:29.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:29.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_43 2026-03-24T02:44:29.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2026-03-24T02:44:29.582 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:29.810 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_43: (2) No such file or directory 2026-03-24T02:44:29.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:29.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:29.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:29.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:29.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:29.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_44 2026-03-24T02:44:29.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_44 deleted 2026-03-24T02:44:29.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:29.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:29.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:44:29.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:29.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:29.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:29.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:29.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_44 2026-03-24T02:44:29.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:29.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:29.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:44:29.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2026-03-24T02:44:29.815 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:29.839 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_44: (2) No such file or directory 2026-03-24T02:44:29.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:29.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:29.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:29.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_44 deleted '' 2026-03-24T02:44:29.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:29.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:29.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_44 2026-03-24T02:44:29.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:29.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:29.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:29.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:29.944 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_44 2026-03-24T02:44:29.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:29.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:29.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_44 2026-03-24T02:44:29.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2026-03-24T02:44:29.945 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:29.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_44: (2) No such file or directory 2026-03-24T02:44:29.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:29.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:29.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:29.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:29.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:29.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_45 2026-03-24T02:44:29.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_45 deleted 2026-03-24T02:44:29.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:29.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:29.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_45 2026-03-24T02:44:29.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:29.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:29.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:29.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:29.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_45 2026-03-24T02:44:29.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:29.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:29.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_45 2026-03-24T02:44:29.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2026-03-24T02:44:29.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:29.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_45: (2) No such file or directory 2026-03-24T02:44:30.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:30.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:30.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:30.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_45 deleted '' 2026-03-24T02:44:30.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:30.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:30.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_45 2026-03-24T02:44:30.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:30.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:30.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:30.101 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:30.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_45 2026-03-24T02:44:30.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:30.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:30.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_45 2026-03-24T02:44:30.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2026-03-24T02:44:30.102 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:30.130 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_45: (2) No such file or directory 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_46 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_46 deleted 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_46 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:30.135 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_46 2026-03-24T02:44:30.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2026-03-24T02:44:30.136 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:30.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_46: (2) No such file or directory 2026-03-24T02:44:30.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:30.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:30.169 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:30.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_46 deleted '' 2026-03-24T02:44:30.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:30.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:30.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_46 2026-03-24T02:44:30.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:30.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:30.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:30.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:30.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_46 2026-03-24T02:44:30.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:30.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:30.271 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_46 2026-03-24T02:44:30.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2026-03-24T02:44:30.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:30.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_46: (2) No such file or directory 2026-03-24T02:44:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_47 2026-03-24T02:44:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_47 deleted 2026-03-24T02:44:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:44:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:30.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:30.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_47 2026-03-24T02:44:30.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:30.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:30.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_47 2026-03-24T02:44:30.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2026-03-24T02:44:30.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:30.328 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_47: (2) No such file or directory 2026-03-24T02:44:30.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:30.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:30.331 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:30.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_47 deleted '' 2026-03-24T02:44:30.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:30.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:30.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_47 2026-03-24T02:44:30.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:30.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:30.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:30.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:30.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_47 2026-03-24T02:44:30.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:30.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:30.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_47 2026-03-24T02:44:30.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2026-03-24T02:44:30.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:30.458 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_47: (2) No such file or directory 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_48 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_48 deleted 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_48 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_48 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_48 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2026-03-24T02:44:30.462 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:30.487 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_48: (2) No such file or directory 2026-03-24T02:44:30.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:30.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:30.492 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:30.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_48 deleted '' 2026-03-24T02:44:30.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:30.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:30.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_48 2026-03-24T02:44:30.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:30.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:30.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:30.593 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:30.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_48 2026-03-24T02:44:30.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:30.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:30.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_48 2026-03-24T02:44:30.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2026-03-24T02:44:30.594 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:30.622 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_48: (2) No such file or directory 2026-03-24T02:44:30.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:30.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:30.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:30.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:30.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:30.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_49 2026-03-24T02:44:30.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_49 deleted 2026-03-24T02:44:30.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:30.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:30.626 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:44:30.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:30.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:30.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:30.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:30.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_49 2026-03-24T02:44:30.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:30.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:30.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:44:30.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2026-03-24T02:44:30.627 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:30.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_49: (2) No such file or directory 2026-03-24T02:44:30.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:30.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:30.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:30.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_49 deleted '' 2026-03-24T02:44:30.758 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:30.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:30.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_49 2026-03-24T02:44:30.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:30.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:30.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:30.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:30.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_49 2026-03-24T02:44:30.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:30.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:30.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_49 2026-03-24T02:44:30.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2026-03-24T02:44:30.759 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:30.784 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_49: (2) No such file or directory 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_50 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_50 deleted 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_50 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_50 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2026-03-24T02:44:30.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:30.814 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_50: (2) No such file or directory 2026-03-24T02:44:30.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:44:30.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:30.817 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:30.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_50 deleted '' 2026-03-24T02:44:30.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:30.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:30.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_50 2026-03-24T02:44:30.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:44:30.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:30.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:30.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:30.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_50 2026-03-24T02:44:30.918 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:30.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:30.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_50 2026-03-24T02:44:30.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2026-03-24T02:44:30.919 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:30.943 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_50: (2) No such file or directory 2026-03-24T02:44:30.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:44:30.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:30.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:44:30.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:30.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ testlog 'TEST: delete images during bootstrap' 2026-03-24T02:44:30.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ tee -a /tmp/tmp.l3GEA596QR/rbd-mirror.test.log 2026-03-24T02:44:30.947 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ date '+%F %T' 2026-03-24T02:44:30.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo 2026-03-24 02:44:30 TEST: delete images during bootstrap 2026-03-24T02:44:30.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:2026-03-24 02:44:30 TEST: delete images during bootstrap 2026-03-24T02:44:30.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_pool_mirror_mode cluster1 mirror image 2026-03-24T02:44:30.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:30.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:30.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=image 2026-03-24T02:44:30.948 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster1 mirror pool enable mirror image 2026-03-24T02:44:30.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:note: changing mirroring mode from pool to image 2026-03-24T02:44:30.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_pool_mirror_mode cluster2 mirror image 2026-03-24T02:44:30.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:30.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:30.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=image 2026-03-24T02:44:30.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror image 2026-03-24T02:44:31.010 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:note: changing mirroring mode from pool to image 2026-03-24T02:44:31.021 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ start_mirror cluster1 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set_cluster_instance cluster1 cluster instance 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local val=cluster1 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster_var_name=cluster 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local instance_var_name=instance 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cluster=cluster1 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=cluster1 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' cluster1 = cluster1 ']' 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ instance=0 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval cluster=cluster1 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ cluster=cluster1 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ eval instance=0 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ instance=0 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 1 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=test 2026-03-24T02:44:31.022 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 1 10 2026-03-24T02:44:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:44:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_1 2026-03-24T02:44:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_1 512M 2026-03-24T02:44:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:44:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:44:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:44:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:44:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:44:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 512M ']' 2026-03-24T02:44:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=512M 2026-03-24T02:44:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:44:31.023 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_1 2026-03-24T02:44:31.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ enable_mirror cluster2 mirror image_1 2026-03-24T02:44:31.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:31.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:31.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:44:31.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:44:31.075 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 journal 2026-03-24T02:44:31.111 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring enabled 2026-03-24T02:44:31.118 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 info mirror/image_1 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_1': 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 0 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 2338660fc9df 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.2338660fc9df 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:44:31 2026 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:44:31 2026 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:44:31 2026 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 2338660fc9df 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: c06518ca-1798-43cc-97f8-b74c75e095ba 2026-03-24T02:44:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: true 2026-03-24T02:44:31.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_1 2026-03-24T02:44:31.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:31.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:31.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:44:31.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:44:31.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=9 2026-03-24T02:44:31.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:44:31.155 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:44:36.720 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 86408192 bytes 2026-03-24T02:44:38.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 100968448 bytes 2026-03-24T02:44:39.355 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8894096681671214) > weight (0.25) 2026-03-24T02:44:39.355 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-03-24T02:44:39.355 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-03-24T02:44:39.355 INFO:tasks.rbd_mirror_thrash:waiting for 6.113112080256865 secs before reviving daemons 2026-03-24T02:44:39.356 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:44:39.353+0000 7f6c54b5a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 76006) UID: 1000 2026-03-24T02:44:40.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:44:40.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:44:40.165 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:44:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_1 present 2026-03-24T02:44:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:44:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=present 2026-03-24T02:44:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:44:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:44:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:44:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_1 2026-03-24T02:44:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:44:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-03-24T02:44:40.166 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:40.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id=121cb022921f 2026-03-24T02:44:40.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:44:40.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:44:40.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_1 present 121cb022921f 2026-03-24T02:44:40.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:44:40.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:40.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:44:40.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=present 2026-03-24T02:44:40.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id=121cb022921f 2026-03-24T02:44:40.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:44:40.300 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:44:40.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_1 2026-03-24T02:44:40.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:44:40.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:44:40.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:44:40.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-03-24T02:44:40.301 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:44:40.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id=121cb022921f 2026-03-24T02:44:40.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 121cb022921f 2026-03-24T02:44:40.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -z 121cb022921f -o 121cb022921f = 121cb022921f 2026-03-24T02:44:40.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=present 2026-03-24T02:44:40.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test present = present 2026-03-24T02:44:40.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:44:40.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ disable_mirror cluster2 mirror image_1 2026-03-24T02:44:40.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:44:40.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:44:40.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:44:40.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_1 2026-03-24T02:44:43.354 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:44:43.349+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e609200 finish: resending after timeout 2026-03-24T02:44:44.531 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.39925252980928183) > weight (0.25) 2026-03-24T02:44:44.531 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7339898041206094) > weight (0.25) 2026-03-24T02:44:44.531 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.8282826909902926) > weight (0.25) 2026-03-24T02:44:44.531 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-03-24T02:44:44.531 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-03-24T02:44:44.531 INFO:tasks.rbd_mirror_thrash:waiting for 5.265965379769003 secs before reviving daemons 2026-03-24T02:44:44.532 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:44:44.525+0000 7fe4f3034640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 74362) UID: 1000 2026-03-24T02:44:45.469 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-03-24T02:44:45.469 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:44:45.469 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-03-24T02:44:45.469 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-03-24T02:44:45.470 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-03-24T02:44:45.471 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-03-24T02:44:45.471 INFO:tasks.rbd_mirror_thrash:waiting for 116.64778698878251 secs before thrashing 2026-03-24T02:44:48.354 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:44:48.349+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e609200 finish: resending after timeout 2026-03-24T02:44:49.798 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-03-24T02:44:49.798 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:44:49.798 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-03-24T02:44:49.798 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-03-24T02:44:49.799 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-03-24T02:44:49.800 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-03-24T02:44:49.801 INFO:tasks.rbd_mirror_thrash:waiting for 95.3046823841193 secs before thrashing 2026-03-24T02:44:53.355 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:44:53.353+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e609200 finish: resending after timeout 2026-03-24T02:44:58.356 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:44:58.353+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e609200 finish: resending after timeout 2026-03-24T02:45:03.356 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:45:03.353+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e609200 finish: resending after timeout 2026-03-24T02:45:06.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring disabled 2026-03-24T02:45:06.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2026-03-24T02:45:06.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:45:06.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:45:06.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:45:06.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:45:06.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:45:06.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:45:06.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:45:06.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_1 2026-03-24T02:45:06.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:45:06.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:45:06.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:45:06.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-03-24T02:45:06.874 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:45:06.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id=121cb022921f 2026-03-24T02:45:06.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:45:06.910 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:45:07.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_1 deleted 121cb022921f 2026-03-24T02:45:07.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:45:07.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:45:07.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:45:07.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:45:07.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id=121cb022921f 2026-03-24T02:45:07.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:45:07.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:45:07.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_1 2026-03-24T02:45:07.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:45:07.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:45:07.011 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:45:07.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-03-24T02:45:07.012 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:45:07.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id=121cb022921f 2026-03-24T02:45:07.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 121cb022921f 2026-03-24T02:45:07.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -z 121cb022921f -o 121cb022921f = 121cb022921f 2026-03-24T02:45:07.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=present 2026-03-24T02:45:07.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = present 2026-03-24T02:45:07.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:45:07.045 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 1 2026-03-24T02:45:08.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_1 deleted 121cb022921f 2026-03-24T02:45:08.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:45:08.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:45:08.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:45:08.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:45:08.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id=121cb022921f 2026-03-24T02:45:08.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:45:08.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:45:08.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_1 2026-03-24T02:45:08.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:45:08.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:45:08.046 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_1 2026-03-24T02:45:08.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-03-24T02:45:08.047 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:45:08.079 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-03-24T02:45:08.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:45:08.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:45:08.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:45:08.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:45:08.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ purge_snapshots cluster2 mirror image_1 2026-03-24T02:45:08.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:45:08.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:45:08.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:45:08.085 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap purge mirror/image_1 2026-03-24T02:45:08.791 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-03-24T02:45:08.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_1 2026-03-24T02:45:08.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:45:08.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:45:08.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:45:08.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:45:08.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:45:08.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_1 2026-03-24T02:45:08.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:45:08.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:45:08.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_1 2026-03-24T02:45:08.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-03-24T02:45:08.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-03-24T02:45:09.184 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-03-24T02:45:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:45:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:45:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_2 2026-03-24T02:45:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_2 512M 2026-03-24T02:45:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:45:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:45:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:45:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:45:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:45:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:45:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:45:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 512M ']' 2026-03-24T02:45:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=512M 2026-03-24T02:45:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:45:09.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_2 2026-03-24T02:45:09.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ enable_mirror cluster2 mirror image_2 2026-03-24T02:45:09.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:45:09.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:45:09.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:45:09.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:45:09.241 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 journal 2026-03-24T02:45:09.281 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring enabled 2026-03-24T02:45:09.290 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 info mirror/image_2 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_2': 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 0 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 235d2081d421 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.235d2081d421 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:45:09 2026 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:45:09 2026 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:45:09 2026 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 235d2081d421 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: d6db7bf5-b8cf-47aa-867c-2cf765f7ca88 2026-03-24T02:45:09.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: true 2026-03-24T02:45:09.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_2 2026-03-24T02:45:09.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:45:09.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:45:09.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:45:09.325 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:45:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=10 2026-03-24T02:45:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:45:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:45:12.086 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 28092416 bytes 2026-03-24T02:45:17.141 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:45:17.133+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e608600 finish: resending after timeout 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_2 present 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=present 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_2 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:45:19.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-03-24T02:45:19.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id=121c210b12eb 2026-03-24T02:45:19.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:45:19.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:45:19.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_2 present 121c210b12eb 2026-03-24T02:45:19.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:45:19.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:45:19.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:45:19.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=present 2026-03-24T02:45:19.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id=121c210b12eb 2026-03-24T02:45:19.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:45:19.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:45:19.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_2 2026-03-24T02:45:19.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:45:19.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:45:19.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:45:19.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-03-24T02:45:19.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:45:19.576 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id=121c210b12eb 2026-03-24T02:45:19.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 121c210b12eb 2026-03-24T02:45:19.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -z 121c210b12eb -o 121c210b12eb = 121c210b12eb 2026-03-24T02:45:19.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=present 2026-03-24T02:45:19.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test present = present 2026-03-24T02:45:19.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:45:19.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ disable_mirror cluster2 mirror image_2 2026-03-24T02:45:19.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:45:19.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:45:19.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:45:19.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_2 2026-03-24T02:45:22.142 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:45:22.137+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e608600 finish: resending after timeout 2026-03-24T02:45:27.143 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:45:27.137+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e608600 finish: resending after timeout 2026-03-24T02:45:32.143 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:45:32.141+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e608600 finish: resending after timeout 2026-03-24T02:45:37.144 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:45:37.137+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e608600 finish: resending after timeout 2026-03-24T02:45:42.144 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:45:42.141+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e608600 finish: resending after timeout 2026-03-24T02:45:47.145 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:45:47.137+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e608600 finish: resending after timeout 2026-03-24T02:45:52.145 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:45:52.137+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e608600 finish: resending after timeout 2026-03-24T02:45:57.146 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:45:57.137+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e608600 finish: resending after timeout 2026-03-24T02:46:02.147 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:46:02.141+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e608600 finish: resending after timeout 2026-03-24T02:46:03.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring disabled 2026-03-24T02:46:04.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2026-03-24T02:46:04.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:46:04.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:04.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:46:04.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:46:04.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:46:04.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:46:04.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:46:04.003 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:46:03.997+0000 7ff7df364640 -1 rbd::mirror::image_replayer::journal::SyncPointHandler: 0x55e9ecc44cf0 handle_update_sync_points: failed to update remote journal client meta for image d6db7bf5-b8cf-47aa-867c-2cf765f7ca88: (2) No such file or directory 2026-03-24T02:46:04.003 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:46:03.997+0000 7ff7df364640 -1 rbd::mirror::image_sync::SyncPointPruneRequest: 0x55e9e5f4da40 handle_update_sync_points: failed to update client data: (2) No such file or directory 2026-03-24T02:46:04.003 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:46:03.997+0000 7ff7df364640 -1 rbd::mirror::ImageSync: 0x55e9e6106160 handle_prune_sync_points: failed to prune sync point: (2) No such file or directory 2026-03-24T02:46:04.003 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:46:03.997+0000 7ff7df364640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55e9e6163040 handle_image_sync: failed to sync remote image: (2) No such file or directory 2026-03-24T02:46:04.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_2 2026-03-24T02:46:04.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:46:04.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:46:04.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:46:04.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:46:04.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-03-24T02:46:04.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id=121c210b12eb 2026-03-24T02:46:04.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:46:04.051 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:46:04.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_2 deleted 121c210b12eb 2026-03-24T02:46:04.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:46:04.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:04.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:46:04.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:46:04.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id=121c210b12eb 2026-03-24T02:46:04.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:46:04.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:46:04.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_2 2026-03-24T02:46:04.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:46:04.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:46:04.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_2 2026-03-24T02:46:04.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-03-24T02:46:04.152 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:46:04.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-03-24T02:46:04.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:46:04.178 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:46:04.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:46:04.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:46:04.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ purge_snapshots cluster2 mirror image_2 2026-03-24T02:46:04.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:46:04.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:04.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:46:04.179 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap purge mirror/image_2 2026-03-24T02:46:04.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_2 2026-03-24T02:46:04.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:46:04.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:04.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:46:04.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:46:04.212 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:46:04.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_2 2026-03-24T02:46:04.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:46:04.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:04.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_2 2026-03-24T02:46:04.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-03-24T02:46:04.245 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-03-24T02:46:04.516 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-03-24T02:46:04.519 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:46:04.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:46:04.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_3 2026-03-24T02:46:04.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_3 512M 2026-03-24T02:46:04.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:46:04.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:46:04.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:04.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:46:04.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:46:04.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:46:04.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:46:04.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 512M ']' 2026-03-24T02:46:04.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=512M 2026-03-24T02:46:04.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:46:04.520 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_3 2026-03-24T02:46:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ enable_mirror cluster2 mirror image_3 2026-03-24T02:46:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:46:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:46:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:46:04.564 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 journal 2026-03-24T02:46:04.600 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring enabled 2026-03-24T02:46:04.608 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 info mirror/image_3 2026-03-24T02:46:04.634 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_3': 2026-03-24T02:46:04.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:46:04.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:46:04.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 0 2026-03-24T02:46:04.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 23707d7ddd54 2026-03-24T02:46:04.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.23707d7ddd54 2026-03-24T02:46:04.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:46:04.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:46:04.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:46:04.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:46:04.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:46:04 2026 2026-03-24T02:46:04.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:46:04 2026 2026-03-24T02:46:04.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:46:04 2026 2026-03-24T02:46:04.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 23707d7ddd54 2026-03-24T02:46:04.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:46:04.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:46:04.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 34e6dd33-3c6f-4d98-985a-a8884006d18e 2026-03-24T02:46:04.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: true 2026-03-24T02:46:04.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_3 2026-03-24T02:46:04.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:46:04.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:04.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:46:04.639 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:46:04.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=7 2026-03-24T02:46:04.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:46:04.640 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:46:06.991 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 29593600 bytes 2026-03-24T02:46:08.715 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 34953216 bytes 2026-03-24T02:46:11.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 72545280 bytes 2026-03-24T02:46:11.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:46:11.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:46:11.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:46:11.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:46:11.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_3 present 2026-03-24T02:46:11.649 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:46:11.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:11.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:46:11.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=present 2026-03-24T02:46:11.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:46:11.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:46:11.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:46:11.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_3 2026-03-24T02:46:11.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:46:11.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:46:11.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:46:11.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-03-24T02:46:11.650 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:46:11.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id=12f371d16a01 2026-03-24T02:46:11.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:46:11.722 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:46:11.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_3 present 12f371d16a01 2026-03-24T02:46:11.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:46:11.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:11.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:46:11.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=present 2026-03-24T02:46:11.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id=12f371d16a01 2026-03-24T02:46:11.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:46:11.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:46:11.823 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_3 2026-03-24T02:46:11.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:46:11.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:46:11.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:46:11.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-03-24T02:46:11.824 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:46:11.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id=12f371d16a01 2026-03-24T02:46:11.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 12f371d16a01 2026-03-24T02:46:11.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -z 12f371d16a01 -o 12f371d16a01 = 12f371d16a01 2026-03-24T02:46:11.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=present 2026-03-24T02:46:11.863 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test present = present 2026-03-24T02:46:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:46:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ disable_mirror cluster2 mirror image_3 2026-03-24T02:46:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:46:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:46:11.864 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_3 2026-03-24T02:46:25.106 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6591763553161795) > weight (0.25) 2026-03-24T02:46:25.106 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.6668423517715399) > weight (0.25) 2026-03-24T02:46:25.106 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.5467744986472267) > weight (0.25) 2026-03-24T02:46:25.106 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.2722980431259858) > weight (0.25) 2026-03-24T02:46:25.106 INFO:tasks.rbd_mirror_thrash:waiting for 92.23259290515605 secs before thrashing 2026-03-24T02:46:41.868 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring disabled 2026-03-24T02:46:41.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2026-03-24T02:46:41.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:46:41.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:41.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:46:41.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:46:41.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:46:41.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:46:41.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:46:41.876 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_3 2026-03-24T02:46:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:46:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:46:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:46:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-03-24T02:46:41.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:46:41.899 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-03-24T02:46:41.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:46:41.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:46:41.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:46:42.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2026-03-24T02:46:42.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:46:42.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:42.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:46:42.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:46:42.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:46:42.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:46:42.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:46:42.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_3 2026-03-24T02:46:42.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:46:42.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:46:42.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_3 2026-03-24T02:46:42.004 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-03-24T02:46:42.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:46:42.027 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-03-24T02:46:42.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:46:42.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:46:42.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:46:42.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:46:42.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ purge_snapshots cluster2 mirror image_3 2026-03-24T02:46:42.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:46:42.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:42.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:46:42.030 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap purge mirror/image_3 2026-03-24T02:46:42.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_3 2026-03-24T02:46:42.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:46:42.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:42.062 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:46:42.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:46:42.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:46:42.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_3 2026-03-24T02:46:42.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:46:42.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:42.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_3 2026-03-24T02:46:42.063 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-03-24T02:46:42.097 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-03-24T02:46:42.120 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5472471569276438) > weight (0.25) 2026-03-24T02:46:42.120 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.3182390829057571) > weight (0.25) 2026-03-24T02:46:42.120 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.3137491469701963) > weight (0.25) 2026-03-24T02:46:42.120 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.7244826647317685) > weight (0.25) 2026-03-24T02:46:42.120 INFO:tasks.rbd_mirror_thrash:waiting for 88.6294470554319 secs before thrashing 2026-03-24T02:46:42.360 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-03-24T02:46:42.364 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:46:42.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:46:42.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_4 2026-03-24T02:46:42.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_4 512M 2026-03-24T02:46:42.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:46:42.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:46:42.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:42.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:46:42.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:46:42.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:46:42.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:46:42.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 512M ']' 2026-03-24T02:46:42.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=512M 2026-03-24T02:46:42.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:46:42.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_4 2026-03-24T02:46:42.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ enable_mirror cluster2 mirror image_4 2026-03-24T02:46:42.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:46:42.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:42.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:46:42.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:46:42.408 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 journal 2026-03-24T02:46:42.443 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring enabled 2026-03-24T02:46:42.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 info mirror/image_4 2026-03-24T02:46:42.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_4': 2026-03-24T02:46:42.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:46:42.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:46:42.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 0 2026-03-24T02:46:42.478 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 23832b4653d6 2026-03-24T02:46:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.23832b4653d6 2026-03-24T02:46:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:46:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:46:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:46:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:46:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:46:42 2026 2026-03-24T02:46:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:46:42 2026 2026-03-24T02:46:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:46:42 2026 2026-03-24T02:46:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 23832b4653d6 2026-03-24T02:46:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:46:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:46:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 0b8a4f95-49c1-4c12-975f-4c3022bbffad 2026-03-24T02:46:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: true 2026-03-24T02:46:42.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_4 2026-03-24T02:46:42.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:46:42.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:42.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:46:42.482 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:46:42.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=13 2026-03-24T02:46:42.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:46:42.484 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:46:45.465 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 41400320 bytes 2026-03-24T02:46:47.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 62294016 bytes 2026-03-24T02:46:47.862 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 62905344 bytes 2026-03-24T02:46:50.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:400 IOs, 126785536 bytes 2026-03-24T02:46:52.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:500 IOs, 157763584 bytes 2026-03-24T02:46:55.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:46:55.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:46:55.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:46:55.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:46:55.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_4 present 2026-03-24T02:46:55.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:46:55.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:55.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:46:55.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=present 2026-03-24T02:46:55.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:46:55.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:46:55.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:46:55.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_4 2026-03-24T02:46:55.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:46:55.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:46:55.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_4 2026-03-24T02:46:55.503 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-03-24T02:46:55.507 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:46:55.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id=121c60c1b025 2026-03-24T02:46:55.551 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:46:55.552 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:46:55.652 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_4 present 121c60c1b025 2026-03-24T02:46:55.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:46:55.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:55.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:46:55.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=present 2026-03-24T02:46:55.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id=121c60c1b025 2026-03-24T02:46:55.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:46:55.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:46:55.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_4 2026-03-24T02:46:55.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:46:55.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:46:55.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_4 2026-03-24T02:46:55.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-03-24T02:46:55.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:46:55.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id=121c60c1b025 2026-03-24T02:46:55.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 121c60c1b025 2026-03-24T02:46:55.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -z 121c60c1b025 -o 121c60c1b025 = 121c60c1b025 2026-03-24T02:46:55.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=present 2026-03-24T02:46:55.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test present = present 2026-03-24T02:46:55.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:46:55.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ disable_mirror cluster2 mirror image_4 2026-03-24T02:46:55.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:46:55.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:46:55.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:46:55.691 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_4 2026-03-24T02:47:24.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring disabled 2026-03-24T02:47:24.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2026-03-24T02:47:24.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:47:24.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:47:24.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:47:24.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:47:24.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:47:24.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:47:24.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:47:24.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_4 2026-03-24T02:47:24.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:47:24.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:47:24.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_4 2026-03-24T02:47:24.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-03-24T02:47:24.843 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:47:24.867 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-03-24T02:47:24.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:47:24.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:47:24.871 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:47:24.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2026-03-24T02:47:24.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:47:24.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:47:24.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:47:24.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:47:24.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:47:24.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:47:24.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:47:24.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_4 2026-03-24T02:47:24.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:47:24.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:47:24.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_4 2026-03-24T02:47:24.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:47:24.972 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-03-24T02:47:24.997 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-03-24T02:47:25.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:47:25.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:47:25.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:47:25.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:47:25.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ purge_snapshots cluster2 mirror image_4 2026-03-24T02:47:25.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:47:25.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:47:25.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:47:25.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap purge mirror/image_4 2026-03-24T02:47:25.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_4 2026-03-24T02:47:25.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:47:25.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:47:25.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:47:25.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:47:25.034 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:47:25.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_4 2026-03-24T02:47:25.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:47:25.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:47:25.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_4 2026-03-24T02:47:25.035 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-03-24T02:47:25.066 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-03-24T02:47:25.317 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-03-24T02:47:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:47:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:47:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_5 2026-03-24T02:47:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_5 512M 2026-03-24T02:47:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:47:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:47:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:47:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:47:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:47:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:47:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:47:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 512M ']' 2026-03-24T02:47:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=512M 2026-03-24T02:47:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:47:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_5 2026-03-24T02:47:25.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ enable_mirror cluster2 mirror image_5 2026-03-24T02:47:25.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:47:25.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:47:25.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:47:25.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:47:25.365 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 journal 2026-03-24T02:47:25.398 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring enabled 2026-03-24T02:47:25.404 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 info mirror/image_5 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_5': 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 0 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 23983246b52b 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.23983246b52b 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:47:25 2026 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:47:25 2026 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:47:25 2026 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 23983246b52b 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 53fc961b-52dd-4de4-9ca5-be0d588c8cc5 2026-03-24T02:47:25.428 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: true 2026-03-24T02:47:25.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_5 2026-03-24T02:47:25.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:47:25.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:47:25.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:47:25.431 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:47:25.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=6 2026-03-24T02:47:25.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:47:25.432 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:47:26.973 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 24848384 bytes 2026-03-24T02:47:30.042 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 68034560 bytes 2026-03-24T02:47:31.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:47:31.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:47:31.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:47:31.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:47:31.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_5 present 2026-03-24T02:47:31.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:47:31.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:47:31.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:47:31.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=present 2026-03-24T02:47:31.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:47:31.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:47:31.455 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:47:31.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_5 2026-03-24T02:47:31.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:47:31.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:47:31.456 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:47:31.457 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-03-24T02:47:31.463 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:47:31.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id=121c3faad3b8 2026-03-24T02:47:31.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:47:31.541 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:47:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_5 present 121c3faad3b8 2026-03-24T02:47:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:47:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:47:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:47:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=present 2026-03-24T02:47:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id=121c3faad3b8 2026-03-24T02:47:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:47:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:47:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_5 2026-03-24T02:47:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:47:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:47:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:47:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-03-24T02:47:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:47:31.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id=121c3faad3b8 2026-03-24T02:47:31.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 121c3faad3b8 2026-03-24T02:47:31.675 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -z 121c3faad3b8 -o 121c3faad3b8 = 121c3faad3b8 2026-03-24T02:47:31.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=present 2026-03-24T02:47:31.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test present = present 2026-03-24T02:47:31.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:47:31.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ disable_mirror cluster2 mirror image_5 2026-03-24T02:47:31.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:47:31.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:47:31.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:47:31.676 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_5 2026-03-24T02:47:32.484 INFO:tasks.ceph.cluster2.mon.a.vm07.stderr:2026-03-24T02:47:32.477+0000 7fd1fb039640 -1 log_channel(cluster) log [ERR] : Health check failed: mon a is very low on available space (MON_DISK_CRIT) 2026-03-24T02:47:32.980 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:47:32.973+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3dff8a80 finish: resending after timeout 2026-03-24T02:47:34.224 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:47:34.217+0000 7fdb32ec6640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55637cd5ad00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-03-24T02:47:34.224 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:47:34.217+0000 7fdb32ec6640 -1 rbd::mirror::ImageReplayer: 0x55637a943b80 [3/53fc961b-52dd-4de4-9ca5-be0d588c8cc5] operator(): start failed: (85) Interrupted system call should be restarted 2026-03-24T02:47:37.980 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:47:37.977+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3dff8a80 finish: resending after timeout 2026-03-24T02:47:42.981 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:47:42.973+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3dff8a80 finish: resending after timeout 2026-03-24T02:47:47.982 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:47:47.973+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3dff8a80 finish: resending after timeout 2026-03-24T02:47:52.983 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:47:52.977+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3dff8a80 finish: resending after timeout 2026-03-24T02:47:57.340 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6445568847405689) > weight (0.25) 2026-03-24T02:47:57.340 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5705350988182366) > weight (0.25) 2026-03-24T02:47:57.340 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.4058240171256595) > weight (0.25) 2026-03-24T02:47:57.340 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-03-24T02:47:57.340 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-03-24T02:47:57.340 INFO:tasks.rbd_mirror_thrash:waiting for 2.7250123479625743 secs before reviving daemons 2026-03-24T02:47:57.340 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:47:57.333+0000 7fb3d776b640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 85901) UID: 1000 2026-03-24T02:47:57.983 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:47:57.977+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3dff8a80 finish: resending after timeout 2026-03-24T02:48:00.066 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-03-24T02:48:00.066 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:48:00.066 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-03-24T02:48:00.066 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-03-24T02:48:00.066 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-03-24T02:48:00.067 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-03-24T02:48:00.067 INFO:tasks.rbd_mirror_thrash:waiting for 105.97900776198611 secs before thrashing 2026-03-24T02:48:02.984 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:48:02.978+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3dff8a80 finish: resending after timeout 2026-03-24T02:48:04.224 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:48:04.218+0000 7fdb326c5640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55637cd33d40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-03-24T02:48:04.226 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:48:04.222+0000 7fdb326c5640 -1 rbd::mirror::ImageReplayer: 0x55637a943b80 [3/53fc961b-52dd-4de4-9ca5-be0d588c8cc5] operator(): start failed: (85) Interrupted system call should be restarted 2026-03-24T02:48:07.985 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:48:07.978+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3dff8a80 finish: resending after timeout 2026-03-24T02:48:09.681 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:48:09.674+0000 7ff7deb63640 -1 rbd::mirror::image_replayer::journal::SyncPointHandler: 0x55e9e66f4ea0 handle_update_sync_points: failed to update remote journal client meta for image 53fc961b-52dd-4de4-9ca5-be0d588c8cc5: (2) No such file or directory 2026-03-24T02:48:09.681 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:48:09.674+0000 7ff7deb63640 -1 rbd::mirror::image_sync::SyncPointPruneRequest: 0x55e9e6308ea0 handle_update_sync_points: failed to update client data: (2) No such file or directory 2026-03-24T02:48:09.681 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:48:09.674+0000 7ff7deb63640 -1 rbd::mirror::ImageSync: 0x55e9e5ffbb80 handle_prune_sync_points: failed to prune sync point: (2) No such file or directory 2026-03-24T02:48:09.681 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:48:09.674+0000 7ff7deb63640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55e9e594bba0 handle_image_sync: failed to sync remote image: (2) No such file or directory 2026-03-24T02:48:09.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring disabled 2026-03-24T02:48:09.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2026-03-24T02:48:09.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:48:09.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:09.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:48:09.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:48:09.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:48:09.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:48:09.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:48:09.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_5 2026-03-24T02:48:09.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:48:09.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:48:09.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:48:09.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-03-24T02:48:09.772 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:48:09.797 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-03-24T02:48:09.800 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:48:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:48:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:48:09.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2026-03-24T02:48:09.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:48:09.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:09.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:48:09.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:48:09.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:48:09.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:48:09.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:48:09.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_5 2026-03-24T02:48:09.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:48:09.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:48:09.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_5 2026-03-24T02:48:09.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-03-24T02:48:09.903 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:48:09.925 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-03-24T02:48:09.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:48:09.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:48:09.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:48:09.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:48:09.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ purge_snapshots cluster2 mirror image_5 2026-03-24T02:48:09.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:48:09.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:09.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:48:09.928 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap purge mirror/image_5 2026-03-24T02:48:09.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_5 2026-03-24T02:48:09.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:48:09.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:09.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:48:09.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:48:09.961 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:48:09.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_5 2026-03-24T02:48:09.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:48:09.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:09.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_5 2026-03-24T02:48:09.962 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-03-24T02:48:09.993 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-03-24T02:48:10.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-03-24T02:48:10.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:48:10.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:48:10.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_6 2026-03-24T02:48:10.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_6 512M 2026-03-24T02:48:10.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:48:10.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:48:10.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:10.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:48:10.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:48:10.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:48:10.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:48:10.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 512M ']' 2026-03-24T02:48:10.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=512M 2026-03-24T02:48:10.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:48:10.449 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_6 2026-03-24T02:48:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ enable_mirror cluster2 mirror image_6 2026-03-24T02:48:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:48:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:48:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:48:10.494 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 journal 2026-03-24T02:48:10.528 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring enabled 2026-03-24T02:48:10.537 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 info mirror/image_6 2026-03-24T02:48:10.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_6': 2026-03-24T02:48:10.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:48:10.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:48:10.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 0 2026-03-24T02:48:10.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 23b0edbcde3b 2026-03-24T02:48:10.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.23b0edbcde3b 2026-03-24T02:48:10.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:48:10.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:48:10.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:48:10.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:48:10.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:48:10 2026 2026-03-24T02:48:10.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:48:10 2026 2026-03-24T02:48:10.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:48:10 2026 2026-03-24T02:48:10.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 23b0edbcde3b 2026-03-24T02:48:10.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:48:10.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:48:10.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 08534af0-c5df-445f-8d00-db11730aaaee 2026-03-24T02:48:10.563 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: true 2026-03-24T02:48:10.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_6 2026-03-24T02:48:10.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:48:10.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:10.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:48:10.566 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:48:10.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=11 2026-03-24T02:48:10.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:48:10.568 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:48:10.750 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8866166358374921) > weight (0.25) 2026-03-24T02:48:10.750 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.8944325989232235) > weight (0.25) 2026-03-24T02:48:10.750 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.8133435461937327) > weight (0.25) 2026-03-24T02:48:10.750 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.8858916612411365) > weight (0.25) 2026-03-24T02:48:10.750 INFO:tasks.rbd_mirror_thrash:waiting for 60.41212910256682 secs before thrashing 2026-03-24T02:48:13.911 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 49029120 bytes 2026-03-24T02:48:15.157 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 60736512 bytes 2026-03-24T02:48:15.909 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 68230144 bytes 2026-03-24T02:48:18.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:400 IOs, 118653952 bytes 2026-03-24T02:48:20.146 INFO:tasks.ceph.cluster2.osd.1.vm07.stderr:2026-03-24T02:48:20.138+0000 7f2e0c59f640 -1 reset not still connected to 0x56026c60fa00 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_6 present 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=present 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_6 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-03-24T02:48:21.577 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:48:21.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id=12f34fa6d48f 2026-03-24T02:48:21.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:48:21.612 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:48:21.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_6 present 12f34fa6d48f 2026-03-24T02:48:21.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:48:21.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:21.713 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:48:21.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=present 2026-03-24T02:48:21.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id=12f34fa6d48f 2026-03-24T02:48:21.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:48:21.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:48:21.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_6 2026-03-24T02:48:21.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:48:21.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:48:21.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:48:21.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-03-24T02:48:21.714 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:48:21.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id=12f34fa6d48f 2026-03-24T02:48:21.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 12f34fa6d48f 2026-03-24T02:48:21.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -z 12f34fa6d48f -o 12f34fa6d48f = 12f34fa6d48f 2026-03-24T02:48:21.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=present 2026-03-24T02:48:21.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test present = present 2026-03-24T02:48:21.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:48:21.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ disable_mirror cluster2 mirror image_6 2026-03-24T02:48:21.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:48:21.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:21.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:48:21.754 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_6 2026-03-24T02:48:51.950 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring disabled 2026-03-24T02:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2026-03-24T02:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_6 2026-03-24T02:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-03-24T02:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:48:51.982 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-03-24T02:48:51.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:48:51.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:48:51.985 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2026-03-24T02:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_6 2026-03-24T02:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_6 2026-03-24T02:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-03-24T02:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:48:52.109 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-03-24T02:48:52.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:48:52.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:48:52.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:48:52.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:48:52.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ purge_snapshots cluster2 mirror image_6 2026-03-24T02:48:52.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:48:52.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:52.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:48:52.112 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap purge mirror/image_6 2026-03-24T02:48:52.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_6 2026-03-24T02:48:52.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:48:52.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:52.144 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:48:52.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:48:52.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:48:52.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_6 2026-03-24T02:48:52.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:48:52.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:52.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_6 2026-03-24T02:48:52.145 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-03-24T02:48:52.175 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-03-24T02:48:52.414 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-03-24T02:48:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:48:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:48:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_7 2026-03-24T02:48:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_7 512M 2026-03-24T02:48:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:48:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:48:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:48:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:48:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:48:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:48:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 512M ']' 2026-03-24T02:48:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=512M 2026-03-24T02:48:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:48:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_7 2026-03-24T02:48:52.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ enable_mirror cluster2 mirror image_7 2026-03-24T02:48:52.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:48:52.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:52.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:48:52.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:48:52.459 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 journal 2026-03-24T02:48:52.488 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring enabled 2026-03-24T02:48:52.493 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 info mirror/image_7 2026-03-24T02:48:52.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_7': 2026-03-24T02:48:52.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:48:52.545 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:48:52.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 0 2026-03-24T02:48:52.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 23c69e057251 2026-03-24T02:48:52.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.23c69e057251 2026-03-24T02:48:52.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:48:52.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:48:52.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:48:52.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:48:52.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:48:52 2026 2026-03-24T02:48:52.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:48:52 2026 2026-03-24T02:48:52.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:48:52 2026 2026-03-24T02:48:52.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 23c69e057251 2026-03-24T02:48:52.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:48:52.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:48:52.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 38c94f27-17cf-4263-bed4-6d2d432eadf0 2026-03-24T02:48:52.546 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: true 2026-03-24T02:48:52.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_7 2026-03-24T02:48:52.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:48:52.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:48:52.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:48:52.548 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:48:52.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=13 2026-03-24T02:48:52.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:48:52.550 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:48:55.350 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 42263552 bytes 2026-03-24T02:49:00.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 123604992 bytes 2026-03-24T02:49:04.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 175716352 bytes 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_7 present 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=present 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_7 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-03-24T02:49:05.562 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:49:05.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id=121c3c80644f 2026-03-24T02:49:05.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:49:05.661 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:49:05.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_7 present 121c3c80644f 2026-03-24T02:49:05.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:49:05.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:49:05.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:49:05.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=present 2026-03-24T02:49:05.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id=121c3c80644f 2026-03-24T02:49:05.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:49:05.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:49:05.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_7 2026-03-24T02:49:05.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:49:05.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:49:05.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:49:05.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-03-24T02:49:05.763 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:49:05.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id=121c3c80644f 2026-03-24T02:49:05.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 121c3c80644f 2026-03-24T02:49:05.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -z 121c3c80644f -o 121c3c80644f = 121c3c80644f 2026-03-24T02:49:05.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=present 2026-03-24T02:49:05.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test present = present 2026-03-24T02:49:05.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:49:05.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ disable_mirror cluster2 mirror image_7 2026-03-24T02:49:05.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:49:05.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:49:05.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:49:05.799 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_7 2026-03-24T02:49:11.163 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-03-24T02:49:11.163 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-03-24T02:49:11.163 INFO:tasks.rbd_mirror_thrash:waiting for 5.894840948068597 secs before reviving daemons 2026-03-24T02:49:11.164 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:49:11.162+0000 7fb81bfa0640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 67863) UID: 1000 2026-03-24T02:49:17.059 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-03-24T02:49:17.059 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:49:17.059 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-03-24T02:49:17.059 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-03-24T02:49:17.059 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-03-24T02:49:17.061 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-03-24T02:49:17.061 INFO:tasks.rbd_mirror_thrash:waiting for 96.93841743795204 secs before thrashing 2026-03-24T02:49:33.525 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring disabled 2026-03-24T02:49:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2026-03-24T02:49:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:49:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:49:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:49:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:49:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:49:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:49:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:49:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_7 2026-03-24T02:49:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:49:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:49:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:49:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-03-24T02:49:33.533 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:49:33.554 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-03-24T02:49:33.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:49:33.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:49:33.556 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:49:33.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2026-03-24T02:49:33.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:49:33.657 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:49:33.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:49:33.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:49:33.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:49:33.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:49:33.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:49:33.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_7 2026-03-24T02:49:33.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:49:33.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:49:33.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_7 2026-03-24T02:49:33.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-03-24T02:49:33.658 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:49:33.683 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-03-24T02:49:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:49:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:49:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:49:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:49:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ purge_snapshots cluster2 mirror image_7 2026-03-24T02:49:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:49:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:49:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:49:33.686 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap purge mirror/image_7 2026-03-24T02:49:33.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_7 2026-03-24T02:49:33.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:49:33.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:49:33.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:49:33.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:49:33.734 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:49:33.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_7 2026-03-24T02:49:33.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:49:33.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:49:33.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_7 2026-03-24T02:49:33.735 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-03-24T02:49:33.969 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-03-24T02:49:34.224 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-03-24T02:49:34.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:49:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:49:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_8 2026-03-24T02:49:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_8 512M 2026-03-24T02:49:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:49:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:49:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:49:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:49:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:49:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:49:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:49:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 512M ']' 2026-03-24T02:49:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=512M 2026-03-24T02:49:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:49:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_8 2026-03-24T02:49:34.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ enable_mirror cluster2 mirror image_8 2026-03-24T02:49:34.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:49:34.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:49:34.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:49:34.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:49:34.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 journal 2026-03-24T02:49:34.306 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring enabled 2026-03-24T02:49:34.311 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 info mirror/image_8 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_8': 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 0 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 23e11623c27d 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.23e11623c27d 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:49:34 2026 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:49:34 2026 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:49:34 2026 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 23e11623c27d 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: b1f5de1d-e323-4e7e-bced-911142d1420b 2026-03-24T02:49:34.335 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: true 2026-03-24T02:49:34.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_8 2026-03-24T02:49:34.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:49:34.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:49:34.337 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:49:34.338 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:49:34.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=6 2026-03-24T02:49:34.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:49:34.339 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:49:37.453 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 45920256 bytes 2026-03-24T02:49:40.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:49:40.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:49:40.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:49:40.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:49:40.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_8 present 2026-03-24T02:49:40.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:49:40.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:49:40.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:49:40.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=present 2026-03-24T02:49:40.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:49:40.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:49:40.354 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:49:40.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_8 2026-03-24T02:49:40.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:49:40.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:49:40.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:49:40.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-03-24T02:49:40.356 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:49:40.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id=121c74bbf6eb 2026-03-24T02:49:40.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:49:40.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:49:40.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_8 present 121c74bbf6eb 2026-03-24T02:49:40.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:49:40.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:49:40.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:49:40.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=present 2026-03-24T02:49:40.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id=121c74bbf6eb 2026-03-24T02:49:40.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:49:40.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:49:40.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_8 2026-03-24T02:49:40.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:49:40.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:49:40.497 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:49:40.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-03-24T02:49:40.498 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:49:40.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id=121c74bbf6eb 2026-03-24T02:49:40.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 121c74bbf6eb 2026-03-24T02:49:40.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -z 121c74bbf6eb -o 121c74bbf6eb = 121c74bbf6eb 2026-03-24T02:49:40.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=present 2026-03-24T02:49:40.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test present = present 2026-03-24T02:49:40.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:49:40.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ disable_mirror cluster2 mirror image_8 2026-03-24T02:49:40.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:49:40.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:49:40.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:49:40.527 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_8 2026-03-24T02:49:42.428 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:49:42.422+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e0858c0 finish: resending after timeout 2026-03-24T02:49:46.048 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6340814978932134) > weight (0.25) 2026-03-24T02:49:46.048 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.3692178514592488) > weight (0.25) 2026-03-24T02:49:46.048 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.5211023594058862) > weight (0.25) 2026-03-24T02:49:46.048 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.889893753719573) > weight (0.25) 2026-03-24T02:49:46.048 INFO:tasks.rbd_mirror_thrash:waiting for 94.48132387405809 secs before thrashing 2026-03-24T02:49:47.429 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:49:47.422+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e0858c0 finish: resending after timeout 2026-03-24T02:49:52.429 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:49:52.426+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e0858c0 finish: resending after timeout 2026-03-24T02:49:57.430 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:49:57.426+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e0858c0 finish: resending after timeout 2026-03-24T02:50:02.431 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:50:02.426+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e0858c0 finish: resending after timeout 2026-03-24T02:50:04.225 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:50:04.218+0000 7fdb32ec6640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55637cd33520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-03-24T02:50:04.225 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:50:04.218+0000 7fdb32ec6640 -1 rbd::mirror::ImageReplayer: 0x55637cc3cf00 [3/b1f5de1d-e323-4e7e-bced-911142d1420b] operator(): start failed: (85) Interrupted system call should be restarted 2026-03-24T02:50:07.431 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:50:07.430+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e0858c0 finish: resending after timeout 2026-03-24T02:50:12.171 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:50:12.166+0000 7ff7deb63640 -1 rbd::mirror::image_replayer::journal::SyncPointHandler: 0x55e9e6148b40 handle_update_sync_points: failed to update remote journal client meta for image b1f5de1d-e323-4e7e-bced-911142d1420b: (2) No such file or directory 2026-03-24T02:50:12.171 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:50:12.166+0000 7ff7deb63640 -1 rbd::mirror::image_sync::SyncPointPruneRequest: 0x55e9e6956240 handle_update_sync_points: failed to update client data: (2) No such file or directory 2026-03-24T02:50:12.171 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:50:12.166+0000 7ff7deb63640 -1 rbd::mirror::ImageSync: 0x55e9e69b8f20 handle_prune_sync_points: failed to prune sync point: (2) No such file or directory 2026-03-24T02:50:12.171 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:50:12.166+0000 7ff7deb63640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55e9e61629c0 handle_image_sync: failed to sync remote image: (2) No such file or directory 2026-03-24T02:50:12.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring disabled 2026-03-24T02:50:12.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2026-03-24T02:50:12.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:50:12.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:12.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:50:12.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:50:12.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:50:12.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:50:12.272 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:50:12.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_8 2026-03-24T02:50:12.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:50:12.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:50:12.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:50:12.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:50:12.274 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-03-24T02:50:12.316 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-03-24T02:50:12.320 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:50:12.320 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:50:12.320 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:50:12.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2026-03-24T02:50:12.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:50:12.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:12.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:50:12.421 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:50:12.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:50:12.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:50:12.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:50:12.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_8 2026-03-24T02:50:12.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:50:12.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:50:12.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_8 2026-03-24T02:50:12.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-03-24T02:50:12.422 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:50:12.444 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-03-24T02:50:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:50:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:50:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:50:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:50:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ purge_snapshots cluster2 mirror image_8 2026-03-24T02:50:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:50:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:50:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap purge mirror/image_8 2026-03-24T02:50:12.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_8 2026-03-24T02:50:12.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:50:12.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:12.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:50:12.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:50:12.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:50:12.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_8 2026-03-24T02:50:12.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:50:12.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:12.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_8 2026-03-24T02:50:12.479 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-03-24T02:50:12.512 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-03-24T02:50:12.764 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-03-24T02:50:12.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:50:12.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:50:12.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_9 2026-03-24T02:50:12.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_9 512M 2026-03-24T02:50:12.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:50:12.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:50:12.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:12.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:50:12.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:50:12.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:50:12.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:50:12.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 512M ']' 2026-03-24T02:50:12.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=512M 2026-03-24T02:50:12.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:50:12.768 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_9 2026-03-24T02:50:12.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ enable_mirror cluster2 mirror image_9 2026-03-24T02:50:12.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:50:12.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:12.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:50:12.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:50:12.811 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 journal 2026-03-24T02:50:12.846 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring enabled 2026-03-24T02:50:12.852 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 info mirror/image_9 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_9': 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 0 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 23f58c20dde7 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.23f58c20dde7 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:50:12 2026 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:50:12 2026 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:50:12 2026 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 23f58c20dde7 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 76519ef1-3721-4ba3-9628-ee415e8981f7 2026-03-24T02:50:12.877 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: true 2026-03-24T02:50:12.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_9 2026-03-24T02:50:12.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:50:12.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:12.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:50:12.880 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:50:12.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=5 2026-03-24T02:50:12.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:50:12.882 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:50:16.924 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 51778560 bytes 2026-03-24T02:50:17.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:50:17.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:50:17.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:50:17.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:50:17.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_9 present 2026-03-24T02:50:17.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:50:17.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:17.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:50:17.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=present 2026-03-24T02:50:17.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:50:17.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:50:17.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:50:17.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_9 2026-03-24T02:50:17.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:50:17.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:50:17.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:50:17.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-03-24T02:50:17.892 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:50:17.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id=12f319e7cad 2026-03-24T02:50:17.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:50:17.930 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:50:18.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_9 present 12f319e7cad 2026-03-24T02:50:18.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:50:18.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:18.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:50:18.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=present 2026-03-24T02:50:18.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id=12f319e7cad 2026-03-24T02:50:18.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:50:18.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:50:18.031 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_9 2026-03-24T02:50:18.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:50:18.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:50:18.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:50:18.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-03-24T02:50:18.032 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:50:18.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id=12f319e7cad 2026-03-24T02:50:18.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 12f319e7cad 2026-03-24T02:50:18.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -z 12f319e7cad -o 12f319e7cad = 12f319e7cad 2026-03-24T02:50:18.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=present 2026-03-24T02:50:18.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test present = present 2026-03-24T02:50:18.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:50:18.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ disable_mirror cluster2 mirror image_9 2026-03-24T02:50:18.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:50:18.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:18.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:50:18.072 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_9 2026-03-24T02:50:18.970 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:50:18.966+0000 7f2174df3640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x56453d4c1040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-03-24T02:50:18.971 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:50:18.966+0000 7f2174df3640 -1 rbd::mirror::ImageReplayer: 0x56453d915680 [3/76519ef1-3721-4ba3-9628-ee415e8981f7] operator(): start failed: (85) Interrupted system call should be restarted 2026-03-24T02:50:19.509 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:50:19.502+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e084cc0 finish: resending after timeout 2026-03-24T02:50:24.510 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:50:24.502+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e084cc0 finish: resending after timeout 2026-03-24T02:50:29.511 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:50:29.506+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e084cc0 finish: resending after timeout 2026-03-24T02:50:34.511 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:50:34.506+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e084cc0 finish: resending after timeout 2026-03-24T02:50:39.512 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:50:39.506+0000 7fe621d6a640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e084cc0 finish: resending after timeout 2026-03-24T02:50:44.343 INFO:tasks.ceph.cluster2.osd.2.vm07.stderr:2026-03-24T02:50:44.334+0000 7f085e5d7640 -1 reset not still connected to 0x5587b5213d40 2026-03-24T02:50:44.512 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:50:44.506+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55cd3e084cc0 finish: resending after timeout 2026-03-24T02:50:47.126 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:50:47.118+0000 7fe615551640 -1 rbd::mirror::image_replayer::journal::SyncPointHandler: 0x55cd3d1333b0 handle_update_sync_points: failed to update remote journal client meta for image 76519ef1-3721-4ba3-9628-ee415e8981f7: (2) No such file or directory 2026-03-24T02:50:47.126 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:50:47.118+0000 7fe615551640 -1 rbd::mirror::image_sync::SyncPointPruneRequest: 0x55cd48022540 handle_update_sync_points: failed to update client data: (2) No such file or directory 2026-03-24T02:50:47.126 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:50:47.118+0000 7fe615551640 -1 rbd::mirror::ImageSync: 0x55cd3dd234a0 handle_prune_sync_points: failed to prune sync point: (2) No such file or directory 2026-03-24T02:50:47.126 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:50:47.118+0000 7fe615551640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cd3ca07d40 handle_image_sync: failed to sync remote image: (2) No such file or directory 2026-03-24T02:50:47.186 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring disabled 2026-03-24T02:50:47.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2026-03-24T02:50:47.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:50:47.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:47.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:50:47.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:50:47.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:50:47.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:50:47.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:50:47.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_9 2026-03-24T02:50:47.198 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:50:47.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:50:47.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:50:47.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-03-24T02:50:47.199 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:50:47.228 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-03-24T02:50:47.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:50:47.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:50:47.231 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:50:47.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2026-03-24T02:50:47.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:50:47.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:47.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:50:47.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:50:47.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:50:47.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:50:47.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:50:47.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_9 2026-03-24T02:50:47.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:50:47.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:50:47.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_9 2026-03-24T02:50:47.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-03-24T02:50:47.334 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:50:47.358 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-03-24T02:50:47.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:50:47.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:50:47.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:50:47.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:50:47.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ purge_snapshots cluster2 mirror image_9 2026-03-24T02:50:47.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:50:47.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:47.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:50:47.362 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap purge mirror/image_9 2026-03-24T02:50:47.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_9 2026-03-24T02:50:47.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:50:47.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:47.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:50:47.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:50:47.395 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:50:47.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_9 2026-03-24T02:50:47.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:50:47.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:47.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_9 2026-03-24T02:50:47.396 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-03-24T02:50:47.429 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-03-24T02:50:47.891 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-03-24T02:50:47.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:50:47.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for i in `seq 1 10` 2026-03-24T02:50:47.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image=image_10 2026-03-24T02:50:47.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ create_image cluster2 mirror image_10 512M 2026-03-24T02:50:47.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:50:47.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:50:47.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:47.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:50:47.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:50:47.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:50:47.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local size=128 2026-03-24T02:50:47.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -n 512M ']' 2026-03-24T02:50:47.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ size=512M 2026-03-24T02:50:47.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ shift 2026-03-24T02:50:47.896 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_10 2026-03-24T02:50:47.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ enable_mirror cluster2 mirror image_10 2026-03-24T02:50:47.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:50:47.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:47.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:50:47.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local mode=journal 2026-03-24T02:50:47.939 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 journal 2026-03-24T02:50:47.971 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring enabled 2026-03-24T02:50:47.976 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 info mirror/image_10 2026-03-24T02:50:48.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:rbd image 'image_10': 2026-03-24T02:50:48.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: size 512 MiB in 128 objects 2026-03-24T02:50:48.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: order 22 (4 MiB objects) 2026-03-24T02:50:48.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: snapshot_count: 0 2026-03-24T02:50:48.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: id: 2408dbb22751 2026-03-24T02:50:48.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: block_name_prefix: rbd_data.2408dbb22751 2026-03-24T02:50:48.000 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: format: 2 2026-03-24T02:50:48.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: features: layering, exclusive-lock, journaling 2026-03-24T02:50:48.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: op_features: 2026-03-24T02:50:48.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: flags: 2026-03-24T02:50:48.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: create_timestamp: Tue Mar 24 02:50:47 2026 2026-03-24T02:50:48.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: access_timestamp: Tue Mar 24 02:50:47 2026 2026-03-24T02:50:48.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: modify_timestamp: Tue Mar 24 02:50:47 2026 2026-03-24T02:50:48.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: journal: 2408dbb22751 2026-03-24T02:50:48.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring state: enabled 2026-03-24T02:50:48.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring mode: journal 2026-03-24T02:50:48.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring global id: 7d32dd29-8ee1-4679-8667-d2c9463c4fc1 2026-03-24T02:50:48.001 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout: mirroring primary: true 2026-03-24T02:50:48.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stress_write_image cluster2 mirror image_10 2026-03-24T02:50:48.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:50:48.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:48.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:50:48.003 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-03-24T02:50:48.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local duration=8 2026-03-24T02:50:48.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:50:48.005 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2026-03-24T02:50:48.717 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:100 IOs, 7311360 bytes 2026-03-24T02:50:51.653 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:200 IOs, 49795072 bytes 2026-03-24T02:50:54.000 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5247261993851879) > weight (0.25) 2026-03-24T02:50:54.000 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.5258763272133209) > weight (0.25) 2026-03-24T02:50:54.000 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.41793738431929384) > weight (0.25) 2026-03-24T02:50:54.000 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.8645368895608526) > weight (0.25) 2026-03-24T02:50:54.000 INFO:tasks.rbd_mirror_thrash:waiting for 97.83490306586309 secs before thrashing 2026-03-24T02:50:55.635 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:300 IOs, 102898688 bytes 2026-03-24T02:50:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=124 2026-03-24T02:50:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:50:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 124 -eq 124 ']' 2026-03-24T02:50:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:50:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_10 present 2026-03-24T02:50:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:50:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:50:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=present 2026-03-24T02:50:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:50:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:50:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:50:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_10 2026-03-24T02:50:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:50:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:50:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_10 2026-03-24T02:50:56.015 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-03-24T02:50:56.020 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:50:56.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id=121c108a0e8f 2026-03-24T02:50:56.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:50:56.074 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:50:56.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_10 present 121c108a0e8f 2026-03-24T02:50:56.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:50:56.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:56.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:50:56.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=present 2026-03-24T02:50:56.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id=121c108a0e8f 2026-03-24T02:50:56.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:50:56.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:50:56.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_10 2026-03-24T02:50:56.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:50:56.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:50:56.176 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_10 2026-03-24T02:50:56.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-03-24T02:50:56.177 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:50:56.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id=121c108a0e8f 2026-03-24T02:50:56.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 121c108a0e8f 2026-03-24T02:50:56.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -z 121c108a0e8f -o 121c108a0e8f = 121c108a0e8f 2026-03-24T02:50:56.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_state=present 2026-03-24T02:50:56.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test present = present 2026-03-24T02:50:56.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:50:56.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ disable_mirror cluster2 mirror image_10 2026-03-24T02:50:56.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:50:56.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:50:56.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:50:56.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_10 2026-03-24T02:51:20.530 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.49228681675972674) > weight (0.25) 2026-03-24T02:51:20.530 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-03-24T02:51:20.530 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-03-24T02:51:20.530 INFO:tasks.rbd_mirror_thrash:waiting for 0.9060544955587824 secs before reviving daemons 2026-03-24T02:51:20.531 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:20.526+0000 7f9a340de640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 80199) UID: 1000 2026-03-24T02:51:21.437 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-03-24T02:51:21.437 INFO:teuthology.orchestra.run:waiting for 600 2026-03-24T02:51:21.438 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-03-24T02:51:21.438 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-03-24T02:51:21.438 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-03-24T02:51:21.439 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-03-24T02:51:21.439 INFO:tasks.rbd_mirror_thrash:waiting for 106.5610080974776 secs before thrashing 2026-03-24T02:51:24.698 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:Mirroring disabled 2026-03-24T02:51:24.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2026-03-24T02:51:24.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:51:24.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:51:24.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:51:24.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local state=deleted 2026-03-24T02:51:24.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:51:24.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local s 2026-03-24T02:51:24.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:51:24.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_10 2026-03-24T02:51:24.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:51:24.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:51:24.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_10 2026-03-24T02:51:24.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-03-24T02:51:24.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:51:24.727 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-03-24T02:51:24.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ image_id= 2026-03-24T02:51:24.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-24T02:51:24.730 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0.1 2026-03-24T02:51:24.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2026-03-24T02:51:24.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:51:24.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:51:24.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:51:24.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local test_state=deleted 2026-03-24T02:51:24.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image_id= 2026-03-24T02:51:24.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_state=deleted 2026-03-24T02:51:24.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local current_image_id 2026-03-24T02:51:24.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ get_image_id cluster1 mirror image_10 2026-03-24T02:51:24.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:51:24.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:51:24.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local image=image_10 2026-03-24T02:51:24.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-03-24T02:51:24.831 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-24T02:51:24.854 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-03-24T02:51:24.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ current_image_id= 2026-03-24T02:51:24.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n '' 2026-03-24T02:51:24.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test deleted = deleted 2026-03-24T02:51:24.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:51:24.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ purge_snapshots cluster2 mirror image_10 2026-03-24T02:51:24.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:51:24.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:51:24.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:51:24.857 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster cluster2 snap purge mirror/image_10 2026-03-24T02:51:24.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image_retry cluster2 mirror image_10 2026-03-24T02:51:24.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:51:24.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:51:24.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:51:24.886 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-24T02:51:24.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:51:24.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ remove_image cluster2 mirror image_10 2026-03-24T02:51:24.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:51:24.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:51:24.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local image=image_10 2026-03-24T02:51:24.887 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-03-24T02:51:24.920 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-03-24T02:51:25.156 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-03-24T02:51:25.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:51:25.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed' 2026-03-24T02:51:25.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ tee -a /tmp/tmp.l3GEA596QR/rbd-mirror.test.log 2026-03-24T02:51:25.160 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ date '+%F %T' 2026-03-24T02:51:25.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo 2026-03-24 02:51:25 TEST: check if removed 'images'\''' OMAP are removed 2026-03-24T02:51:25.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:2026-03-24 02:51:25 TEST: check if removed images' OMAP are removed 2026-03-24T02:51:25.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_in_omap cluster1 mirror 2026-03-24T02:51:25.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:51:25.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:51:25.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2026-03-24T02:51:25.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:51:25.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:51:25.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local obj_name=rbd_mirroring 2026-03-24T02:51:25.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local filter=status_global 2026-03-24T02:51:25.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-24T02:51:25.162 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:51:25.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:51:25.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2026-03-24T02:51:25.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:51:25.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:51:25.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local obj_name=rbd_mirroring 2026-03-24T02:51:25.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local filter=status_global 2026-03-24T02:51:25.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-03-24T02:51:25.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:51:25.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -c status_global 2026-03-24T02:51:25.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:51:25.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local obj_name=rbd_mirroring 2026-03-24T02:51:25.163 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-03-24T02:51:25.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 0 = 0 2026-03-24T02:51:25.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=0 2026-03-24T02:51:25.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:51:25.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 -eq 0 ']' 2026-03-24T02:51:25.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:51:25.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2026-03-24T02:51:25.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:51:25.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:51:25.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local obj_name=rbd_mirroring 2026-03-24T02:51:25.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local filter=image_ 2026-03-24T02:51:25.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-24T02:51:25.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:51:25.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:51:25.189 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2026-03-24T02:51:25.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:51:25.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:51:25.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local obj_name=rbd_mirroring 2026-03-24T02:51:25.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local filter=image_ 2026-03-24T02:51:25.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-03-24T02:51:25.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:51:25.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -c image_ 2026-03-24T02:51:25.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:51:25.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local obj_name=rbd_mirroring 2026-03-24T02:51:25.190 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-03-24T02:51:25.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 0 = 0 2026-03-24T02:51:25.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=0 2026-03-24T02:51:25.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:51:25.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 -eq 0 ']' 2026-03-24T02:51:25.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:51:25.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2026-03-24T02:51:25.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:51:25.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:51:25.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local obj_name=rbd_mirror_leader 2026-03-24T02:51:25.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local filter=image_map 2026-03-24T02:51:25.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-24T02:51:25.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:51:25.213 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:51:25.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-03-24T02:51:25.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:51:25.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:51:25.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local obj_name=rbd_mirror_leader 2026-03-24T02:51:25.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local filter=image_map 2026-03-24T02:51:25.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-03-24T02:51:25.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster1 2026-03-24T02:51:25.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -c image_map 2026-03-24T02:51:25.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:51:25.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local obj_name=rbd_mirror_leader 2026-03-24T02:51:25.214 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-03-24T02:51:25.236 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 0 = 0 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=0 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 -eq 0 ']' 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_image_in_omap cluster2 mirror 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local obj_name=rbd_mirroring 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local filter=status_global 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local obj_name=rbd_mirroring 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local filter=status_global 2026-03-24T02:51:25.237 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-03-24T02:51:25.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:51:25.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -c status_global 2026-03-24T02:51:25.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:51:25.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local obj_name=rbd_mirroring 2026-03-24T02:51:25.238 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-03-24T02:51:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 0 = 0 2026-03-24T02:51:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=0 2026-03-24T02:51:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:51:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 -eq 0 ']' 2026-03-24T02:51:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:51:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2026-03-24T02:51:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:51:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:51:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local obj_name=rbd_mirroring 2026-03-24T02:51:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local filter=image_ 2026-03-24T02:51:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-24T02:51:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:51:25.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:51:25.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2026-03-24T02:51:25.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:51:25.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:51:25.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local obj_name=rbd_mirroring 2026-03-24T02:51:25.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local filter=image_ 2026-03-24T02:51:25.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-03-24T02:51:25.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:51:25.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -c image_ 2026-03-24T02:51:25.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:51:25.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local obj_name=rbd_mirroring 2026-03-24T02:51:25.260 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-03-24T02:51:25.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 0 = 0 2026-03-24T02:51:25.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=0 2026-03-24T02:51:25.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:51:25.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 -eq 0 ']' 2026-03-24T02:51:25.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:51:25.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2026-03-24T02:51:25.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:51:25.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local pool=mirror 2026-03-24T02:51:25.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local obj_name=rbd_mirror_leader 2026-03-24T02:51:25.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local filter=image_map 2026-03-24T02:51:25.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-24T02:51:25.282 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ sleep 0 2026-03-24T02:51:25.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:51:25.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2026-03-24T02:51:25.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:51:25.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:51:25.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local obj_name=rbd_mirror_leader 2026-03-24T02:51:25.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local filter=image_map 2026-03-24T02:51:25.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2026-03-24T02:51:25.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local cluster=cluster2 2026-03-24T02:51:25.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local pool=mirror 2026-03-24T02:51:25.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ local obj_name=rbd_mirror_leader 2026-03-24T02:51:25.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ grep -c image_map 2026-03-24T02:51:25.283 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2026-03-24T02:51:25.304 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test 0 = 0 2026-03-24T02:51:25.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ error_code=0 2026-03-24T02:51:25.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set -e 2026-03-24T02:51:25.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 -eq 0 ']' 2026-03-24T02:51:25.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 0 2026-03-24T02:51:25.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ cleanup 0 2026-03-24T02:51:25.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local error_code=0 2026-03-24T02:51:25.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ set +e 2026-03-24T02:51:25.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 -ne 0 ']' 2026-03-24T02:51:25.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -z '' ']' 2026-03-24T02:51:25.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster instance 2026-03-24T02:51:25.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ CEPH_ARGS= 2026-03-24T02:51:25.305 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-03-24T02:51:25.692 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:pool 'mirror' does not exist 2026-03-24T02:51:25.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ CEPH_ARGS= 2026-03-24T02:51:25.705 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-03-24T02:51:26.731 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:pool 'mirror' does not exist 2026-03-24T02:51:26.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ CEPH_ARGS= 2026-03-24T02:51:26.744 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-03-24T02:51:27.697 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:pool 'mirror_parent' does not exist 2026-03-24T02:51:27.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ CEPH_ARGS= 2026-03-24T02:51:27.710 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-03-24T02:51:28.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:27.998+0000 7fdb32ec6640 -1 rbd::mirror::LeaderWatcher: 0x55637c707500 handle_notify_heartbeat: error notifying heartbeat: (2) No such file or directory, releasing leader 2026-03-24T02:51:28.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:27.998+0000 7fdb2beb8640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55637bf24480 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-03-24T02:51:28.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:27.998+0000 7fdb2beb8640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55637bf25680 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-03-24T02:51:28.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:27.998+0000 7fdb2beb8640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55637bf258c0 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-03-24T02:51:28.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:27.998+0000 7fdb2beb8640 -1 rbd::mirror::PoolWatcher: 0x55637c0883c0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:28.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:27.998+0000 7fdb2c6b9640 -1 rbd::mirror::PoolWatcher: 0x55637c088000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:28.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:27.998+0000 7fdb2c6b9640 -1 rbd::mirror::PoolWatcher: 0x55637c0881e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:28.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:27.998+0000 7fdb256ab640 -1 rbd::mirror::PoolWatcher: 0x55637c0885a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:28.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:27.998+0000 7fdb256ab640 -1 rbd::mirror::PoolWatcher: 0x55637c0890e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:28.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:27.998+0000 7fdb256ab640 -1 rbd::mirror::PoolWatcher: 0x55637c088d20 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:28.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:27.998+0000 7fdb326c5640 -1 rbd::mirror::LeaderWatcher: 0x55637c707500 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-03-24T02:51:28.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:27.998+0000 7fdb326c5640 -1 rbd::mirror::LeaderWatcher: 0x55637c707500 handle_release_leader_lock: error releasing lock: (2) No such file or directory 2026-03-24T02:51:28.775 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:pool 'mirror_parent' does not exist 2026-03-24T02:51:28.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-03-24T02:51:28.787 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stop_mirrors cluster1 2026-03-24T02:51:28.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1 2026-03-24T02:51:28.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local sig= 2026-03-24T02:51:28.788 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stop_mirror cluster1:0 '' 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:0 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local sig= 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 1 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stop_mirror cluster1:1 '' 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:1 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local sig= 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 1 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stop_mirror cluster1:2 '' 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:2 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local sig= 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 1 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stop_mirror cluster1:3 '' 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster1:3 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local sig= 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 1 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stop_mirrors cluster2 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local sig= 2026-03-24T02:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:++ seq 0 3 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stop_mirror cluster2:0 '' 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2:0 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local sig= 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 1 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stop_mirror cluster2:1 '' 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2:1 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local sig= 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 1 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stop_mirror cluster2:2 '' 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2:2 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local sig= 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 1 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ stop_mirror cluster2:3 '' 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local cluster=cluster2:3 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ local sig= 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test -n 1 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ return 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' -z 1 ']' 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ test '' = /tmp/tmp.l3GEA596QR 2026-03-24T02:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ rm -Rf /tmp/tmp.l3GEA596QR 2026-03-24T02:51:28.792 INFO:tasks.workunit.cluster1.client.mirror.vm07.stdout:OK 2026-03-24T02:51:28.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ '[' 0 -eq 0 ']' 2026-03-24T02:51:28.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ echo OK 2026-03-24T02:51:28.793 INFO:tasks.workunit.cluster1.client.mirror.vm07.stderr:+ exit 0 2026-03-24T02:51:28.794 INFO:teuthology.orchestra.run:Running command with timeout 3600 2026-03-24T02:51:28.794 DEBUG:teuthology.orchestra.run.vm07:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2026-03-24T02:51:28.849 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror_stress.sh'] on cluster1.client.mirror... 2026-03-24T02:51:28.849 DEBUG:teuthology.orchestra.run.vm07:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2026-03-24T02:51:29.220 DEBUG:teuthology.parallel:result is None 2026-03-24T02:51:29.225 DEBUG:teuthology.orchestra.run.vm07:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-03-24T02:51:29.233 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-03-24T02:51:29.233 DEBUG:teuthology.orchestra.run.vm07:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-03-24T02:51:29.278 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-03-24T02:51:29.280 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2026-03-24T02:51:29.308 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2026-03-24T02:51:29.309 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 17 2026-03-24T02:51:29.309 INFO:tasks.rbd_mirror_thrash:done joining 2026-03-24T02:51:29.309 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2026-03-24T02:51:29.313 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2026-03-24T02:51:29.313 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 14 2026-03-24T02:51:29.313 INFO:tasks.rbd_mirror_thrash:done joining 2026-03-24T02:51:29.313 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-03-24T02:51:29.318 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.3:waiting for process to exit 2026-03-24T02:51:29.318 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:51:29.324 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.318+0000 7fdb336c7640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 26456) UID: 1000 2026-03-24T02:51:29.325 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.318+0000 7fdb32ec6640 -1 rbd::mirror::InstanceWatcher: 0x55637bf26380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4304 object: (2) No such file or directory 2026-03-24T02:51:29.326 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.318+0000 7fdb32ec6640 -1 rbd::mirror::InstanceWatcher: 0x55637bf26700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4304 object: (2) No such file or directory 2026-03-24T02:51:29.326 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.318+0000 7fdb32ec6640 -1 rbd::mirror::InstanceWatcher: 0x55637c677c00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4304 object: (2) No such file or directory 2026-03-24T02:51:29.326 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb2c6b9640 -1 rbd::mirror::InstanceWatcher: 0x55637bf26380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.326 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb2c6b9640 -1 rbd::mirror::InstanceWatcher: 0x55637bf26700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.326 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb2c6b9640 -1 rbd::mirror::InstanceWatcher: 0x55637c677c00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.327 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb25eac640 -1 rbd::mirror::MirrorStatusUpdater 0x55637bccaa20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.327 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb25eac640 -1 rbd::mirror::MirrorStatusUpdater 0x55637bcca7e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.327 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb25eac640 -1 rbd::mirror::MirrorStatusUpdater 0x55637bccb0e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.327 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb32ec6640 -1 rbd::mirror::NamespaceReplayer: 0x55637bba4540 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.327 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb32ec6640 -1 rbd::mirror::NamespaceReplayer: 0x55637bba4700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.327 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb32ec6640 -1 rbd::mirror::NamespaceReplayer: 0x55637bba48c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.327 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb2c6b9640 -1 rbd::mirror::MirrorStatusUpdater 0x55637bcca480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.327 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb2c6b9640 -1 rbd::mirror::MirrorStatusUpdater 0x55637bcca5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.327 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb2c6b9640 -1 rbd::mirror::MirrorStatusUpdater 0x55637bcca6c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.327 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb326c5640 -1 rbd::mirror::NamespaceReplayer: 0x55637bba4540 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.327 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb326c5640 -1 rbd::mirror::NamespaceReplayer: 0x55637bba4700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.327 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb326c5640 -1 rbd::mirror::NamespaceReplayer: 0x55637bba48c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb1ee9e640 -1 rbd::mirror::PoolWatcher: 0x55637c0894a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb1ee9e640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55637bf25d40 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb17e90640 -1 rbd::mirror::PoolWatcher: 0x55637a870d20 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb18691640 -1 rbd::mirror::PoolWatcher: 0x55637c088b40 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.322+0000 7fdb1e69d640 -1 rbd::mirror::PoolWatcher: 0x55637c088960 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.326+0000 7fdb326c5640 -1 rbd::mirror::InstanceWatcher: 0x55637bf26a80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4324 object: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.326+0000 7fdb326c5640 -1 rbd::mirror::InstanceWatcher: 0x55637bf26e00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4324 object: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.326+0000 7fdb1e69d640 -1 rbd::mirror::InstanceWatcher: 0x55637bf26a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.326+0000 7fdb17e90640 -1 rbd::mirror::MirrorStatusUpdater 0x55637c074480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.326+0000 7fdb32ec6640 -1 rbd::mirror::NamespaceReplayer: 0x55637bba4c40 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.326+0000 7fdb1e69d640 -1 rbd::mirror::InstanceWatcher: 0x55637bf26e00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.326+0000 7fdb1e69d640 -1 rbd::mirror::MirrorStatusUpdater 0x55637bccb9e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.326+0000 7fdb32ec6640 -1 rbd::mirror::NamespaceReplayer: 0x55637bba4c40 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.326+0000 7fdb18691640 -1 rbd::mirror::MirrorStatusUpdater 0x55637bccb560 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.326+0000 7fdb32ec6640 -1 rbd::mirror::NamespaceReplayer: 0x55637bba4e00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.326+0000 7fdb1ee9e640 -1 rbd::mirror::MirrorStatusUpdater 0x55637bccbb00 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.326+0000 7fdb32ec6640 -1 rbd::mirror::NamespaceReplayer: 0x55637bba4e00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.330+0000 7fdb32ec6640 -1 rbd::mirror::LeaderWatcher: 0x55637bd31500 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.330+0000 7fdb32ec6640 -1 rbd::mirror::LeaderWatcher: 0x55637bd31500 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2026-03-24T02:51:29.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.330+0000 7fdb326c5640 -1 rbd::mirror::LeaderWatcher: 0x55637bd31500 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-03-24T02:51:29.358 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.354+0000 7fdb326c5640 -1 rbd::mirror::LeaderWatcher: 0x55637c707500 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-03-24T02:51:29.374 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Stopped 2026-03-24T02:51:29.374 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-03-24T02:51:29.377 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.2:waiting for process to exit 2026-03-24T02:51:29.377 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:51:29.418 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f21755f4640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 60094) UID: 1000 2026-03-24T02:51:29.419 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f2174df3640 -1 rbd::mirror::InstanceWatcher: 0x56453db92700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6947 object: (2) No such file or directory 2026-03-24T02:51:29.419 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f2174df3640 -1 rbd::mirror::InstanceWatcher: 0x56453d2d6380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6947 object: (2) No such file or directory 2026-03-24T02:51:29.419 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f2174df3640 -1 rbd::mirror::InstanceWatcher: 0x56453db92a80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6940 object: (2) No such file or directory 2026-03-24T02:51:29.419 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f2174df3640 -1 rbd::mirror::InstanceWatcher: 0x56453d7bc380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6940 object: (2) No such file or directory 2026-03-24T02:51:29.419 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f2174df3640 -1 rbd::mirror::InstanceWatcher: 0x56453db92380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6940 object: (2) No such file or directory 2026-03-24T02:51:29.419 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f2160dcb640 -1 rbd::mirror::InstanceWatcher: 0x56453db92700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.419 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f2160dcb640 -1 rbd::mirror::InstanceWatcher: 0x56453d2d6380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.420 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f2160dcb640 -1 rbd::mirror::MirrorStatusUpdater 0x56453d2d07e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.420 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f2174df3640 -1 rbd::mirror::NamespaceReplayer: 0x56453ce40e00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.420 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f216dde5640 -1 rbd::mirror::InstanceWatcher: 0x56453d7bc380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.420 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f216dde5640 -1 rbd::mirror::InstanceWatcher: 0x56453db92380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.420 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f216dde5640 -1 rbd::mirror::InstanceWatcher: 0x56453db92a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.420 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f215a5be640 -1 rbd::mirror::MirrorStatusUpdater 0x56453d2d0a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.420 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f2174df3640 -1 rbd::mirror::NamespaceReplayer: 0x56453ce40c40 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.420 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f21605ca640 -1 rbd::mirror::MirrorStatusUpdater 0x56453d2d06c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.420 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f21745f2640 -1 rbd::mirror::NamespaceReplayer: 0x56453ce40c40 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.420 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f2167dd9640 -1 rbd::mirror::MirrorStatusUpdater 0x56453cdd0480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.420 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f2167dd9640 -1 rbd::mirror::MirrorStatusUpdater 0x56453cdd0900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.420 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f2167dd9640 -1 rbd::mirror::MirrorStatusUpdater 0x56453cdd07e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.420 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f2174df3640 -1 rbd::mirror::NamespaceReplayer: 0x56453ce40380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.420 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f2174df3640 -1 rbd::mirror::NamespaceReplayer: 0x56453ce401c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.420 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f2174df3640 -1 rbd::mirror::NamespaceReplayer: 0x56453ce40000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.421 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f2174df3640 -1 rbd::mirror::LeaderWatcher: 0x56453cf86c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-03-24T02:51:29.421 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f216dde5640 -1 rbd::mirror::MirrorStatusUpdater 0x56453cdd0240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.421 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f216dde5640 -1 rbd::mirror::MirrorStatusUpdater 0x56453d966000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.421 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f21745f2640 -1 rbd::mirror::NamespaceReplayer: 0x56453ce401c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.421 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f21745f2640 -1 rbd::mirror::NamespaceReplayer: 0x56453ce40000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.421 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f216e5e6640 -1 rbd::mirror::MirrorStatusUpdater 0x56453cdd0360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.421 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.414+0000 7f21745f2640 -1 rbd::mirror::NamespaceReplayer: 0x56453ce40380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.423 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.418+0000 7f21745f2640 -1 rbd::mirror::LeaderWatcher: 0x56453db7e300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-03-24T02:51:29.435 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Stopped 2026-03-24T02:51:29.435 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-03-24T02:51:29.438 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.1:waiting for process to exit 2026-03-24T02:51:29.438 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:51:29.479 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f5905fbb640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 85810) UID: 1000 2026-03-24T02:51:29.479 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f59057ba640 -1 rbd::mirror::InstanceWatcher: 0x5570147c2380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.9036 object: (2) No such file or directory 2026-03-24T02:51:29.479 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f59057ba640 -1 rbd::mirror::InstanceWatcher: 0x5570147c2a80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.9036 object: (2) No such file or directory 2026-03-24T02:51:29.479 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f5904fb9640 -1 rbd::mirror::InstanceWatcher: 0x557014546700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.9033 object: (2) No such file or directory 2026-03-24T02:51:29.479 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f5904fb9640 -1 rbd::mirror::InstanceWatcher: 0x557014546380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.9033 object: (2) No such file or directory 2026-03-24T02:51:29.479 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f5904fb9640 -1 rbd::mirror::InstanceWatcher: 0x5570147c2700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.9033 object: (2) No such file or directory 2026-03-24T02:51:29.479 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f58f1792640 -1 rbd::mirror::InstanceWatcher: 0x5570147c2380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f58f1792640 -1 rbd::mirror::InstanceWatcher: 0x5570147c2a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f58fe7ac640 -1 rbd::mirror::InstanceWatcher: 0x557014546700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f58fe7ac640 -1 rbd::mirror::InstanceWatcher: 0x557014546380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f58fe7ac640 -1 rbd::mirror::InstanceWatcher: 0x5570147c2700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f58ea784640 -1 rbd::mirror::MirrorStatusUpdater 0x55701458afc0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f58ea784640 -1 rbd::mirror::MirrorStatusUpdater 0x55701458a360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f59057ba640 -1 rbd::mirror::NamespaceReplayer: 0x557014330c40 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f59057ba640 -1 rbd::mirror::NamespaceReplayer: 0x557014330380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f58f87a0640 -1 rbd::mirror::MirrorStatusUpdater 0x55701432e5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f58f87a0640 -1 rbd::mirror::MirrorStatusUpdater 0x55701432e360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f58f87a0640 -1 rbd::mirror::MirrorStatusUpdater 0x55701432e480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f59057ba640 -1 rbd::mirror::NamespaceReplayer: 0x557012fa0e00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f59057ba640 -1 rbd::mirror::NamespaceReplayer: 0x557014330000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f59057ba640 -1 rbd::mirror::NamespaceReplayer: 0x5570143301c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f58f1792640 -1 rbd::mirror::MirrorStatusUpdater 0x55701458ad80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f58f1792640 -1 rbd::mirror::MirrorStatusUpdater 0x55701458ac60 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f59057ba640 -1 rbd::mirror::NamespaceReplayer: 0x557014330c40 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f59057ba640 -1 rbd::mirror::NamespaceReplayer: 0x557014330380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f58fe7ac640 -1 rbd::mirror::MirrorStatusUpdater 0x55701432e000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f58fe7ac640 -1 rbd::mirror::MirrorStatusUpdater 0x55701432e120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f58fe7ac640 -1 rbd::mirror::MirrorStatusUpdater 0x55701432e240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f5904fb9640 -1 rbd::mirror::NamespaceReplayer: 0x557012fa0e00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f5904fb9640 -1 rbd::mirror::NamespaceReplayer: 0x557014330000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.480 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f5904fb9640 -1 rbd::mirror::NamespaceReplayer: 0x5570143301c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.481 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.474+0000 7f59057ba640 -1 rbd::mirror::LeaderWatcher: 0x5570145f7b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-03-24T02:51:29.483 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.478+0000 7f5904fb9640 -1 rbd::mirror::LeaderWatcher: 0x557014078300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-03-24T02:51:29.493 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Stopped 2026-03-24T02:51:29.493 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-03-24T02:51:29.496 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.0:waiting for process to exit 2026-03-24T02:51:29.496 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:51:29.535 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea9a476640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 87960) UID: 1000 2026-03-24T02:51:29.535 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea99c75640 -1 rbd::mirror::InstanceWatcher: 0x5558c85a6700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.9177 object: (2) No such file or directory 2026-03-24T02:51:29.535 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea99c75640 -1 rbd::mirror::InstanceWatcher: 0x5558c85a6000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.9177 object: (2) No such file or directory 2026-03-24T02:51:29.535 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea92c67640 -1 rbd::mirror::InstanceWatcher: 0x5558c81efc00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.535 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea92c67640 -1 rbd::mirror::InstanceWatcher: 0x5558c81ef880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.535 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea93468640 -1 rbd::mirror::InstanceWatcher: 0x5558c85a6380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.535 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea85c4d640 -1 rbd::mirror::InstanceWatcher: 0x5558c85a6700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.535 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea85c4d640 -1 rbd::mirror::InstanceWatcher: 0x5558c85a6000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea8c45a640 -1 rbd::mirror::MirrorStatusUpdater 0x5558c7fd65a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea99c75640 -1 rbd::mirror::NamespaceReplayer: 0x5558c6c56e00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea8c45a640 -1 rbd::mirror::MirrorStatusUpdater 0x5558c7fd6360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea99474640 -1 rbd::mirror::NamespaceReplayer: 0x5558c7fdc1c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea8cc5b640 -1 rbd::mirror::MirrorStatusUpdater 0x5558c7fd6480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea99474640 -1 rbd::mirror::NamespaceReplayer: 0x5558c7fdc000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea7ec3f640 -1 rbd::mirror::MirrorStatusUpdater 0x5558c821d0e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea7ec3f640 -1 rbd::mirror::MirrorStatusUpdater 0x5558c821c480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea92c67640 -1 rbd::mirror::MirrorStatusUpdater 0x5558c7fd6000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea99c75640 -1 rbd::mirror::NamespaceReplayer: 0x5558c7fdc380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea99c75640 -1 rbd::mirror::NamespaceReplayer: 0x5558c7fdc700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea99c75640 -1 rbd::mirror::NamespaceReplayer: 0x5558c6c56e00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea93468640 -1 rbd::mirror::MirrorStatusUpdater 0x5558c7fd6120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea99c75640 -1 rbd::mirror::NamespaceReplayer: 0x5558c7fdc000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea8544c640 -1 rbd::mirror::MirrorStatusUpdater 0x5558c821cb40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea85c4d640 -1 rbd::mirror::MirrorStatusUpdater 0x5558c821cc60 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea99c75640 -1 rbd::mirror::NamespaceReplayer: 0x5558c7fdc380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea99c75640 -1 rbd::mirror::NamespaceReplayer: 0x5558c7fdc700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.530+0000 7fea99c75640 -1 rbd::mirror::LeaderWatcher: 0x5558c8506900 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-03-24T02:51:29.538 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.534+0000 7fea99c75640 -1 rbd::mirror::LeaderWatcher: 0x5558c7e2c300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-03-24T02:51:29.552 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Stopped 2026-03-24T02:51:29.553 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-03-24T02:51:29.556 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.3:waiting for process to exit 2026-03-24T02:51:29.556 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:51:29.560 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.554+0000 7fe62256b640 -1 rbd::mirror::LeaderWatcher: 0x55cd3d84c300 handle_notify_heartbeat: error notifying heartbeat: (2) No such file or directory, releasing leader 2026-03-24T02:51:29.560 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.554+0000 7fe61b55d640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55cd3d396d80 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-03-24T02:51:29.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.554+0000 7fe61b55d640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55cd3d339b00 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-03-24T02:51:29.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.554+0000 7fe61bd5e640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55cd3d3978c0 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-03-24T02:51:29.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.554+0000 7f3eb27b7640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 87254) UID: 1000 2026-03-24T02:51:29.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.554+0000 7fe61bd5e640 -1 rbd::mirror::PoolWatcher: 0x55cd3dab1c20 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:29.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.554+0000 7fe61bd5e640 -1 rbd::mirror::PoolWatcher: 0x55cd3dab0f00 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:29.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.554+0000 7fe61bd5e640 -1 rbd::mirror::PoolWatcher: 0x55cd3dab1a40 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:29.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.554+0000 7fe615551640 -1 rbd::mirror::PoolWatcher: 0x55cd3dab0b40 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:29.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.554+0000 7fe615551640 -1 rbd::mirror::PoolWatcher: 0x55cd3dab0000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:29.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.554+0000 7fe615551640 -1 rbd::mirror::PoolWatcher: 0x55cd3dab0960 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:29.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.554+0000 7f3eb17b5640 -1 rbd::mirror::InstanceWatcher: 0x5643d546b880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5395 object: (2) No such file or directory 2026-03-24T02:51:29.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.554+0000 7f3eb17b5640 -1 rbd::mirror::InstanceWatcher: 0x5643d586a380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5395 object: (2) No such file or directory 2026-03-24T02:51:29.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.554+0000 7f3eb1fb6640 -1 rbd::mirror::InstanceWatcher: 0x5643d5d51500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5392 object: (2) No such file or directory 2026-03-24T02:51:29.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.554+0000 7f3eb1fb6640 -1 rbd::mirror::InstanceWatcher: 0x5643d546b500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5392 object: (2) No such file or directory 2026-03-24T02:51:29.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.554+0000 7f3eb1fb6640 -1 rbd::mirror::InstanceWatcher: 0x5643d5d51180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5392 object: (2) No such file or directory 2026-03-24T02:51:29.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.554+0000 7fe62256b640 -1 rbd::mirror::LeaderWatcher: 0x55cd3d84c300 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-03-24T02:51:29.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.554+0000 7fe62256b640 -1 rbd::mirror::LeaderWatcher: 0x55cd3d84c300 handle_release_leader_lock: error releasing lock: (2) No such file or directory 2026-03-24T02:51:29.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.554+0000 7f3e9d78d640 -1 rbd::mirror::InstanceWatcher: 0x5643d546b880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.554+0000 7f3e9d78d640 -1 rbd::mirror::InstanceWatcher: 0x5643d586a380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.554+0000 7f3eab7a9640 -1 rbd::mirror::InstanceWatcher: 0x5643d5d51500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.554+0000 7f3eab7a9640 -1 rbd::mirror::InstanceWatcher: 0x5643d546b500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.554+0000 7f3eab7a9640 -1 rbd::mirror::InstanceWatcher: 0x5643d5d51180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3e96f80640 -1 rbd::mirror::MirrorStatusUpdater 0x5643d56e9320 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3e96f80640 -1 rbd::mirror::MirrorStatusUpdater 0x5643d56e8d80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3eb1fb6640 -1 rbd::mirror::NamespaceReplayer: 0x5643d551c700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3eb1fb6640 -1 rbd::mirror::NamespaceReplayer: 0x5643d551c8c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3e9d78d640 -1 rbd::mirror::MirrorStatusUpdater 0x5643d56e8a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3e9d78d640 -1 rbd::mirror::MirrorStatusUpdater 0x5643d56e8b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3eb17b5640 -1 rbd::mirror::NamespaceReplayer: 0x5643d551c700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3eb17b5640 -1 rbd::mirror::NamespaceReplayer: 0x5643d551c8c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3ea479b640 -1 rbd::mirror::MirrorStatusUpdater 0x5643d53a4360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3ea479b640 -1 rbd::mirror::MirrorStatusUpdater 0x5643d53a4480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3eb17b5640 -1 rbd::mirror::NamespaceReplayer: 0x5643d4252a80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3eb17b5640 -1 rbd::mirror::NamespaceReplayer: 0x5643d4252000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3ea4f9c640 -1 rbd::mirror::MirrorStatusUpdater 0x5643d53a45a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3eb17b5640 -1 rbd::mirror::NamespaceReplayer: 0x5643d4252700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3eab7a9640 -1 rbd::mirror::MirrorStatusUpdater 0x5643d53a4240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3eb17b5640 -1 rbd::mirror::NamespaceReplayer: 0x5643d4252a80 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3eab7a9640 -1 rbd::mirror::MirrorStatusUpdater 0x5643d53a4120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3eab7a9640 -1 rbd::mirror::MirrorStatusUpdater 0x5643d53a4000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3eb17b5640 -1 rbd::mirror::NamespaceReplayer: 0x5643d4252700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3eb17b5640 -1 rbd::mirror::LeaderWatcher: 0x5643d5628900 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-03-24T02:51:29.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3eb17b5640 -1 rbd::mirror::NamespaceReplayer: 0x5643d4252000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.565 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.vm07.stderr:2026-03-24T02:51:29.558+0000 7f3eb17b5640 -1 rbd::mirror::LeaderWatcher: 0x5643d6084300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-03-24T02:51:29.577 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Stopped 2026-03-24T02:51:29.577 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-03-24T02:51:29.581 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.2:waiting for process to exit 2026-03-24T02:51:29.581 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:51:29.623 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ecb7f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 50999) UID: 1000 2026-03-24T02:51:29.623 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7d8356640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55e9e61d0b40 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-03-24T02:51:29.623 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7d8356640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55e9e61d0240 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-03-24T02:51:29.624 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7d7b55640 -1 rbd::mirror::PoolWatcher: 0x55e9e622d680 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:29.624 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7d7b55640 -1 rbd::mirror::PoolWatcher: 0x55e9edadb4a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:29.624 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ec37e640 -1 rbd::mirror::InstanceWatcher: 0x55e9e608ca80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4636 object: (2) No such file or directory 2026-03-24T02:51:29.624 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7d1348640 -1 rbd::mirror::PoolWatcher: 0x55e9edadab40 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:29.624 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7d1348640 -1 rbd::mirror::PoolWatcher: 0x55e9edadb680 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-03-24T02:51:29.624 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ec37e640 -1 rbd::mirror::InstanceWatcher: 0x55e9e608c000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4636 object: (2) No such file or directory 2026-03-24T02:51:29.624 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ec37e640 -1 rbd::mirror::InstanceWatcher: 0x55e9e608ce00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4636 object: (2) No such file or directory 2026-03-24T02:51:29.624 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7e5b71640 -1 rbd::mirror::InstanceWatcher: 0x55e9e608ca80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7e5b71640 -1 rbd::mirror::InstanceWatcher: 0x55e9e608c000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7e5b71640 -1 rbd::mirror::InstanceWatcher: 0x55e9e608ce00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ec37e640 -1 rbd::mirror::InstanceWatcher: 0x55e9e608c700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4639 object: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ec37e640 -1 rbd::mirror::InstanceWatcher: 0x55e9e608c380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4639 object: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7df364640 -1 rbd::mirror::MirrorStatusUpdater 0x55e9e5caf0e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7df364640 -1 rbd::mirror::MirrorStatusUpdater 0x55e9e5caf8c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7df364640 -1 rbd::mirror::MirrorStatusUpdater 0x55e9e5caf680 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ec37e640 -1 rbd::mirror::NamespaceReplayer: 0x55e9e5b64fc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ec37e640 -1 rbd::mirror::NamespaceReplayer: 0x55e9e5b65180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ec37e640 -1 rbd::mirror::NamespaceReplayer: 0x55e9e5b65340 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7e5b71640 -1 rbd::mirror::MirrorStatusUpdater 0x55e9e5caea20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7e5b71640 -1 rbd::mirror::MirrorStatusUpdater 0x55e9e5caeb40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7d7b55640 -1 rbd::mirror::InstanceWatcher: 0x55e9e608c700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7e5b71640 -1 rbd::mirror::MirrorStatusUpdater 0x55e9e5caec60 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7d7b55640 -1 rbd::mirror::InstanceWatcher: 0x55e9e608c380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ebb7d640 -1 rbd::mirror::NamespaceReplayer: 0x55e9e5b64fc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ebb7d640 -1 rbd::mirror::NamespaceReplayer: 0x55e9e5b65180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ebb7d640 -1 rbd::mirror::NamespaceReplayer: 0x55e9e5b65340 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7d1348640 -1 rbd::mirror::MirrorStatusUpdater 0x55e9e5caf7a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.625 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7d1348640 -1 rbd::mirror::MirrorStatusUpdater 0x55e9e5caf200 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.626 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ebb7d640 -1 rbd::mirror::NamespaceReplayer: 0x55e9e5b65500 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.626 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ebb7d640 -1 rbd::mirror::NamespaceReplayer: 0x55e9e5b656c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.626 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7d8356640 -1 rbd::mirror::MirrorStatusUpdater 0x55e9e5caed80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.626 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7d8356640 -1 rbd::mirror::MirrorStatusUpdater 0x55e9e5caeea0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.626 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ebb7d640 -1 rbd::mirror::NamespaceReplayer: 0x55e9e5b65500 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.626 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ebb7d640 -1 rbd::mirror::NamespaceReplayer: 0x55e9e5b656c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.626 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ec37e640 -1 rbd::mirror::LeaderWatcher: 0x55e9e5d45b00 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-03-24T02:51:29.626 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.618+0000 7ff7ec37e640 -1 rbd::mirror::LeaderWatcher: 0x55e9e5d45b00 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2026-03-24T02:51:29.626 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.622+0000 7ff7ebb7d640 -1 rbd::mirror::LeaderWatcher: 0x55e9e5d45b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-03-24T02:51:29.629 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.vm07.stderr:2026-03-24T02:51:29.622+0000 7ff7ec37e640 -1 rbd::mirror::LeaderWatcher: 0x55e9e676a300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-03-24T02:51:29.645 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Stopped 2026-03-24T02:51:29.645 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-03-24T02:51:29.648 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.1:waiting for process to exit 2026-03-24T02:51:29.648 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:51:29.686 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f884158e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 88906) UID: 1000 2026-03-24T02:51:29.687 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f8840d8d640 -1 rbd::mirror::InstanceWatcher: 0x5608a3bac700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5467 object: (2) No such file or directory 2026-03-24T02:51:29.687 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f8840d8d640 -1 rbd::mirror::InstanceWatcher: 0x5608a3a90a80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5467 object: (2) No such file or directory 2026-03-24T02:51:29.687 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f883a580640 -1 rbd::mirror::InstanceWatcher: 0x5608a3a90000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f883a580640 -1 rbd::mirror::InstanceWatcher: 0x5608a3a90700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f8839d7f640 -1 rbd::mirror::InstanceWatcher: 0x5608a3a90380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f882c564640 -1 rbd::mirror::InstanceWatcher: 0x5608a3bac700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f882c564640 -1 rbd::mirror::InstanceWatcher: 0x5608a3a90a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f8833d73640 -1 rbd::mirror::MirrorStatusUpdater 0x5608a380c7e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f8833d73640 -1 rbd::mirror::MirrorStatusUpdater 0x5608a380c5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f8833d73640 -1 rbd::mirror::MirrorStatusUpdater 0x5608a380c6c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f8840d8d640 -1 rbd::mirror::NamespaceReplayer: 0x5608a2725180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f8840d8d640 -1 rbd::mirror::NamespaceReplayer: 0x5608a2724a80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f8840d8d640 -1 rbd::mirror::NamespaceReplayer: 0x5608a2725340 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f8826558640 -1 rbd::mirror::MirrorStatusUpdater 0x5608a3bd50e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f8826558640 -1 rbd::mirror::MirrorStatusUpdater 0x5608a3bd4b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f8840d8d640 -1 rbd::mirror::NamespaceReplayer: 0x5608a39f08c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f8840d8d640 -1 rbd::mirror::NamespaceReplayer: 0x5608a39f0a80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f882c564640 -1 rbd::mirror::MirrorStatusUpdater 0x5608a3bd47e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f882c564640 -1 rbd::mirror::MirrorStatusUpdater 0x5608a3bd4900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f8840d8d640 -1 rbd::mirror::NamespaceReplayer: 0x5608a39f08c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f8840d8d640 -1 rbd::mirror::NamespaceReplayer: 0x5608a39f0a80 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.689 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.vm07.stderr:2026-03-24T02:51:29.682+0000 7f8840d8d640 -1 rbd::mirror::LeaderWatcher: 0x5608a3b10900 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-03-24T02:51:29.701 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Stopped 2026-03-24T02:51:29.701 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-03-24T02:51:29.705 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.0:waiting for process to exit 2026-03-24T02:51:29.705 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:51:29.746 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe622d6c640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 68715) UID: 1000 2026-03-24T02:51:29.747 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: 0x55cd3cdee700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4854 object: (2) No such file or directory 2026-03-24T02:51:29.747 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: 0x55cd3d150380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4854 object: (2) No such file or directory 2026-03-24T02:51:29.747 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: 0x55cd3cdee000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4851 object: (2) No such file or directory 2026-03-24T02:51:29.747 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: 0x55cd3cdee380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4851 object: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe62256b640 -1 rbd::mirror::InstanceWatcher: 0x55cd3d630380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4851 object: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe60e543640 -1 rbd::mirror::InstanceWatcher: 0x55cd3cdee700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe60e543640 -1 rbd::mirror::InstanceWatcher: 0x55cd3d150380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe61b55d640 -1 rbd::mirror::InstanceWatcher: 0x55cd3cdee000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe61b55d640 -1 rbd::mirror::InstanceWatcher: 0x55cd3d630380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe61b55d640 -1 rbd::mirror::InstanceWatcher: 0x55cd3cdee380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe607d36640 -1 rbd::mirror::MirrorStatusUpdater 0x55cd3cca0d80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe607d36640 -1 rbd::mirror::MirrorStatusUpdater 0x55cd3cca0b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe62256b640 -1 rbd::mirror::NamespaceReplayer: 0x55cd3cdf0700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe614d50640 -1 rbd::mirror::MirrorStatusUpdater 0x55cd3cca0480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe614d50640 -1 rbd::mirror::MirrorStatusUpdater 0x55cd3cca05a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe614d50640 -1 rbd::mirror::MirrorStatusUpdater 0x55cd3cca0360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe62256b640 -1 rbd::mirror::NamespaceReplayer: 0x55cd3cdf08c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe62256b640 -1 rbd::mirror::NamespaceReplayer: 0x55cd3bb48fc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe62256b640 -1 rbd::mirror::NamespaceReplayer: 0x55cd3bb48700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe62256b640 -1 rbd::mirror::NamespaceReplayer: 0x55cd3bb48000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe60dd42640 -1 rbd::mirror::MirrorStatusUpdater 0x55cd3cf32a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe62256b640 -1 rbd::mirror::NamespaceReplayer: 0x55cd3cdf0700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe61bd5e640 -1 rbd::mirror::MirrorStatusUpdater 0x55cd3cca0000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe62256b640 -1 rbd::mirror::NamespaceReplayer: 0x55cd3bb48fc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe61bd5e640 -1 rbd::mirror::MirrorStatusUpdater 0x55cd3cca0240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe60dd42640 -1 rbd::mirror::MirrorStatusUpdater 0x55cd3cf32b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe61b55d640 -1 rbd::mirror::MirrorStatusUpdater 0x55cd3cca0120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe62256b640 -1 rbd::mirror::NamespaceReplayer: 0x55cd3bb48700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe62256b640 -1 rbd::mirror::NamespaceReplayer: 0x55cd3cdf08c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe62256b640 -1 rbd::mirror::NamespaceReplayer: 0x55cd3bb48000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-03-24T02:51:29.748 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.742+0000 7fe62256b640 -1 rbd::mirror::LeaderWatcher: 0x55cd3cf9fb00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-03-24T02:51:29.751 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.vm07.stderr:2026-03-24T02:51:29.746+0000 7fe621d6a640 -1 rbd::mirror::LeaderWatcher: 0x55cd3d84c300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-03-24T02:51:29.766 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Stopped 2026-03-24T02:51:29.766 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-03-24T02:51:29.770 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-03-24T02:51:29.771 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-03-24T02:51:29.973 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:51:29.974 INFO:teuthology.orchestra.run.vm07.stderr:dumped all 2026-03-24T02:51:29.984 INFO:teuthology.orchestra.run.vm07.stdout:{"pg_ready":true,"pg_map":{"version":1526,"stamp":"2026-03-24T02:51:29.208573+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":459299,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":59,"num_write_kb":586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35,"ondisk_log_size":35,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2887556,"kb_used_data":14708,"kb_used_omap":402,"kb_used_meta":2872365,"kb_avail":280227964,"statfs":{"total":289910292480,"available":286953435136,"internally_reserved":0,"allocated":15060992,"data_stored":8063801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":411912,"internal_metadata":2941302520},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-515511927,"num_objects":-182,"num_object_clones":-4,"num_object_copies":-364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-182,"num_whiteouts":-4,"num_read":-659057,"num_read_kb":-47999777,"num_write":-648459,"num_write_kb":-19170673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-8,"num_bytes_recovered":-23294588,"num_keys_recovered":-4,"num_objects_omap":-19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.000800"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":801,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.637533+0000","last_change":"2026-03-24T02:13:56.291116+0000","last_active":"2026-03-24T02:51:24.637533+0000","last_peered":"2026-03-24T02:51:24.637533+0000","last_clean":"2026-03-24T02:51:24.637533+0000","last_became_active":"2026-03-24T02:13:54.285179+0000","last_became_peered":"2026-03-24T02:13:54.285179+0000","last_unstale":"2026-03-24T02:51:24.637533+0000","last_undegraded":"2026-03-24T02:51:24.637533+0000","last_fullsized":"2026-03-24T02:51:24.637533+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:40:35.710968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052393000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":801,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.637449+0000","last_change":"2026-03-24T02:13:56.291113+0000","last_active":"2026-03-24T02:51:24.637449+0000","last_peered":"2026-03-24T02:51:24.637449+0000","last_clean":"2026-03-24T02:51:24.637449+0000","last_became_active":"2026-03-24T02:13:54.286185+0000","last_became_peered":"2026-03-24T02:13:54.286185+0000","last_unstale":"2026-03-24T02:51:24.637449+0000","last_undegraded":"2026-03-24T02:51:24.637449+0000","last_fullsized":"2026-03-24T02:51:24.637449+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:50:07.713703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041187999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":801,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.637381+0000","last_change":"2026-03-24T02:13:56.291271+0000","last_active":"2026-03-24T02:51:24.637381+0000","last_peered":"2026-03-24T02:51:24.637381+0000","last_clean":"2026-03-24T02:51:24.637381+0000","last_became_active":"2026-03-24T02:13:54.284043+0000","last_became_peered":"2026-03-24T02:13:54.284043+0000","last_unstale":"2026-03-24T02:51:24.637381+0000","last_undegraded":"2026-03-24T02:51:24.637381+0000","last_fullsized":"2026-03-24T02:51:24.637381+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:42:40.008023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045964999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":801,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.637308+0000","last_change":"2026-03-24T02:13:56.291308+0000","last_active":"2026-03-24T02:51:24.637308+0000","last_peered":"2026-03-24T02:51:24.637308+0000","last_clean":"2026-03-24T02:51:24.637308+0000","last_became_active":"2026-03-24T02:13:54.283892+0000","last_became_peered":"2026-03-24T02:13:54.283892+0000","last_unstale":"2026-03-24T02:51:24.637308+0000","last_undegraded":"2026-03-24T02:51:24.637308+0000","last_fullsized":"2026-03-24T02:51:24.637308+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T08:17:26.341514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039982800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":804,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.633292+0000","last_change":"2026-03-24T02:13:56.288111+0000","last_active":"2026-03-24T02:51:24.633292+0000","last_peered":"2026-03-24T02:51:24.633292+0000","last_clean":"2026-03-24T02:51:24.633292+0000","last_became_active":"2026-03-24T02:13:54.283188+0000","last_became_peered":"2026-03-24T02:13:54.283188+0000","last_unstale":"2026-03-24T02:51:24.633292+0000","last_undegraded":"2026-03-24T02:51:24.633292+0000","last_fullsized":"2026-03-24T02:51:24.633292+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:58:21.144364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031652200000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":804,"reported_epoch":397,"state":"active+clean","last_fresh":"2026-03-24T02:51:26.656219+0000","last_change":"2026-03-24T02:13:56.736302+0000","last_active":"2026-03-24T02:51:26.656219+0000","last_peered":"2026-03-24T02:51:26.656219+0000","last_clean":"2026-03-24T02:51:26.656219+0000","last_became_active":"2026-03-24T02:13:54.284823+0000","last_became_peered":"2026-03-24T02:13:54.284823+0000","last_unstale":"2026-03-24T02:51:26.656219+0000","last_undegraded":"2026-03-24T02:51:26.656219+0000","last_fullsized":"2026-03-24T02:51:26.656219+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T12:50:46.136937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000322784,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":804,"reported_epoch":397,"state":"active+clean","last_fresh":"2026-03-24T02:51:26.655980+0000","last_change":"2026-03-24T02:13:56.736294+0000","last_active":"2026-03-24T02:51:26.655980+0000","last_peered":"2026-03-24T02:51:26.655980+0000","last_clean":"2026-03-24T02:51:26.655980+0000","last_became_active":"2026-03-24T02:13:54.284987+0000","last_became_peered":"2026-03-24T02:13:54.284987+0000","last_unstale":"2026-03-24T02:51:26.655980+0000","last_undegraded":"2026-03-24T02:51:26.655980+0000","last_fullsized":"2026-03-24T02:51:26.655980+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:52:45.132362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025170100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":803,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.636909+0000","last_change":"2026-03-24T02:13:56.291437+0000","last_active":"2026-03-24T02:51:24.636909+0000","last_peered":"2026-03-24T02:51:24.636909+0000","last_clean":"2026-03-24T02:51:24.636909+0000","last_became_active":"2026-03-24T02:13:54.283277+0000","last_became_peered":"2026-03-24T02:13:54.283277+0000","last_unstale":"2026-03-24T02:51:24.636909+0000","last_undegraded":"2026-03-24T02:51:24.636909+0000","last_fullsized":"2026-03-24T02:51:24.636909+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T10:25:36.882503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039834499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'32","reported_seq":845,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.636875+0000","last_change":"2026-03-24T02:13:52.693536+0000","last_active":"2026-03-24T02:51:24.636875+0000","last_peered":"2026-03-24T02:51:24.636875+0000","last_clean":"2026-03-24T02:51:24.636875+0000","last_became_active":"2026-03-24T02:13:52.693283+0000","last_became_peered":"2026-03-24T02:13:52.693283+0000","last_unstale":"2026-03-24T02:51:24.636875+0000","last_undegraded":"2026-03-24T02:51:24.636875+0000","last_fullsized":"2026-03-24T02:51:24.636875+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:51.152553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:51.152553+0000","last_clean_scrub_stamp":"2026-03-24T02:13:51.152553+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:26:45.574238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":925696,"data_stored":918560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32,"ondisk_log_size":32,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738822,"num_pgs":80,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":840972,"kb_used_data":4588,"kb_used_omap":111,"kb_used_meta":836240,"kb_avail":93530868,"statfs":{"total":96636764160,"available":95775608832,"internally_reserved":0,"allocated":4698112,"data_stored":2368043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":114282,"internal_metadata":856310166},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738821,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1509192,"kb_used_data":5040,"kb_used_omap":141,"kb_used_meta":1503986,"kb_avail":92862648,"statfs":{"total":96636764160,"available":95091351552,"internally_reserved":0,"allocated":5160960,"data_stored":2831495,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":145407,"internal_metadata":1540081665},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738824,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":537392,"kb_used_data":5080,"kb_used_omap":148,"kb_used_meta":532139,"kb_avail":93834448,"statfs":{"total":96636764160,"available":96086474752,"internally_reserved":0,"allocated":5201920,"data_stored":2864263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":152223,"internal_metadata":544910689},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-24T02:51:29.987 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-03-24T02:51:30.153 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:51:30.153 INFO:teuthology.orchestra.run.vm07.stderr:dumped all 2026-03-24T02:51:30.167 INFO:teuthology.orchestra.run.vm07.stdout:{"pg_ready":true,"pg_map":{"version":1526,"stamp":"2026-03-24T02:51:29.208573+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":459299,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":59,"num_write_kb":586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35,"ondisk_log_size":35,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2887556,"kb_used_data":14708,"kb_used_omap":402,"kb_used_meta":2872365,"kb_avail":280227964,"statfs":{"total":289910292480,"available":286953435136,"internally_reserved":0,"allocated":15060992,"data_stored":8063801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":411912,"internal_metadata":2941302520},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-515511927,"num_objects":-182,"num_object_clones":-4,"num_object_copies":-364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-182,"num_whiteouts":-4,"num_read":-659057,"num_read_kb":-47999777,"num_write":-648459,"num_write_kb":-19170673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-8,"num_bytes_recovered":-23294588,"num_keys_recovered":-4,"num_objects_omap":-19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.000800"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":801,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.637533+0000","last_change":"2026-03-24T02:13:56.291116+0000","last_active":"2026-03-24T02:51:24.637533+0000","last_peered":"2026-03-24T02:51:24.637533+0000","last_clean":"2026-03-24T02:51:24.637533+0000","last_became_active":"2026-03-24T02:13:54.285179+0000","last_became_peered":"2026-03-24T02:13:54.285179+0000","last_unstale":"2026-03-24T02:51:24.637533+0000","last_undegraded":"2026-03-24T02:51:24.637533+0000","last_fullsized":"2026-03-24T02:51:24.637533+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:40:35.710968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052393000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":801,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.637449+0000","last_change":"2026-03-24T02:13:56.291113+0000","last_active":"2026-03-24T02:51:24.637449+0000","last_peered":"2026-03-24T02:51:24.637449+0000","last_clean":"2026-03-24T02:51:24.637449+0000","last_became_active":"2026-03-24T02:13:54.286185+0000","last_became_peered":"2026-03-24T02:13:54.286185+0000","last_unstale":"2026-03-24T02:51:24.637449+0000","last_undegraded":"2026-03-24T02:51:24.637449+0000","last_fullsized":"2026-03-24T02:51:24.637449+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:50:07.713703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041187999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":801,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.637381+0000","last_change":"2026-03-24T02:13:56.291271+0000","last_active":"2026-03-24T02:51:24.637381+0000","last_peered":"2026-03-24T02:51:24.637381+0000","last_clean":"2026-03-24T02:51:24.637381+0000","last_became_active":"2026-03-24T02:13:54.284043+0000","last_became_peered":"2026-03-24T02:13:54.284043+0000","last_unstale":"2026-03-24T02:51:24.637381+0000","last_undegraded":"2026-03-24T02:51:24.637381+0000","last_fullsized":"2026-03-24T02:51:24.637381+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:42:40.008023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045964999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":801,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.637308+0000","last_change":"2026-03-24T02:13:56.291308+0000","last_active":"2026-03-24T02:51:24.637308+0000","last_peered":"2026-03-24T02:51:24.637308+0000","last_clean":"2026-03-24T02:51:24.637308+0000","last_became_active":"2026-03-24T02:13:54.283892+0000","last_became_peered":"2026-03-24T02:13:54.283892+0000","last_unstale":"2026-03-24T02:51:24.637308+0000","last_undegraded":"2026-03-24T02:51:24.637308+0000","last_fullsized":"2026-03-24T02:51:24.637308+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T08:17:26.341514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039982800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":804,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.633292+0000","last_change":"2026-03-24T02:13:56.288111+0000","last_active":"2026-03-24T02:51:24.633292+0000","last_peered":"2026-03-24T02:51:24.633292+0000","last_clean":"2026-03-24T02:51:24.633292+0000","last_became_active":"2026-03-24T02:13:54.283188+0000","last_became_peered":"2026-03-24T02:13:54.283188+0000","last_unstale":"2026-03-24T02:51:24.633292+0000","last_undegraded":"2026-03-24T02:51:24.633292+0000","last_fullsized":"2026-03-24T02:51:24.633292+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:58:21.144364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031652200000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":804,"reported_epoch":397,"state":"active+clean","last_fresh":"2026-03-24T02:51:26.656219+0000","last_change":"2026-03-24T02:13:56.736302+0000","last_active":"2026-03-24T02:51:26.656219+0000","last_peered":"2026-03-24T02:51:26.656219+0000","last_clean":"2026-03-24T02:51:26.656219+0000","last_became_active":"2026-03-24T02:13:54.284823+0000","last_became_peered":"2026-03-24T02:13:54.284823+0000","last_unstale":"2026-03-24T02:51:26.656219+0000","last_undegraded":"2026-03-24T02:51:26.656219+0000","last_fullsized":"2026-03-24T02:51:26.656219+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T12:50:46.136937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000322784,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":804,"reported_epoch":397,"state":"active+clean","last_fresh":"2026-03-24T02:51:26.655980+0000","last_change":"2026-03-24T02:13:56.736294+0000","last_active":"2026-03-24T02:51:26.655980+0000","last_peered":"2026-03-24T02:51:26.655980+0000","last_clean":"2026-03-24T02:51:26.655980+0000","last_became_active":"2026-03-24T02:13:54.284987+0000","last_became_peered":"2026-03-24T02:13:54.284987+0000","last_unstale":"2026-03-24T02:51:26.655980+0000","last_undegraded":"2026-03-24T02:51:26.655980+0000","last_fullsized":"2026-03-24T02:51:26.655980+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:52:45.132362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025170100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":803,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.636909+0000","last_change":"2026-03-24T02:13:56.291437+0000","last_active":"2026-03-24T02:51:24.636909+0000","last_peered":"2026-03-24T02:51:24.636909+0000","last_clean":"2026-03-24T02:51:24.636909+0000","last_became_active":"2026-03-24T02:13:54.283277+0000","last_became_peered":"2026-03-24T02:13:54.283277+0000","last_unstale":"2026-03-24T02:51:24.636909+0000","last_undegraded":"2026-03-24T02:51:24.636909+0000","last_fullsized":"2026-03-24T02:51:24.636909+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T10:25:36.882503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039834499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'32","reported_seq":845,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.636875+0000","last_change":"2026-03-24T02:13:52.693536+0000","last_active":"2026-03-24T02:51:24.636875+0000","last_peered":"2026-03-24T02:51:24.636875+0000","last_clean":"2026-03-24T02:51:24.636875+0000","last_became_active":"2026-03-24T02:13:52.693283+0000","last_became_peered":"2026-03-24T02:13:52.693283+0000","last_unstale":"2026-03-24T02:51:24.636875+0000","last_undegraded":"2026-03-24T02:51:24.636875+0000","last_fullsized":"2026-03-24T02:51:24.636875+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:51.152553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:51.152553+0000","last_clean_scrub_stamp":"2026-03-24T02:13:51.152553+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:26:45.574238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":925696,"data_stored":918560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32,"ondisk_log_size":32,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738822,"num_pgs":80,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":840972,"kb_used_data":4588,"kb_used_omap":111,"kb_used_meta":836240,"kb_avail":93530868,"statfs":{"total":96636764160,"available":95775608832,"internally_reserved":0,"allocated":4698112,"data_stored":2368043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":114282,"internal_metadata":856310166},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738821,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1509192,"kb_used_data":5040,"kb_used_omap":141,"kb_used_meta":1503986,"kb_avail":92862648,"statfs":{"total":96636764160,"available":95091351552,"internally_reserved":0,"allocated":5160960,"data_stored":2831495,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":145407,"internal_metadata":1540081665},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738824,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":537392,"kb_used_data":5080,"kb_used_omap":148,"kb_used_meta":532139,"kb_avail":93834448,"statfs":{"total":96636764160,"available":96086474752,"internally_reserved":0,"allocated":5201920,"data_stored":2864263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":152223,"internal_metadata":544910689},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-24T02:51:30.168 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-03-24T02:51:30.168 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-03-24T02:51:30.339 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:51:30.339 INFO:teuthology.orchestra.run.vm07.stderr:dumped all 2026-03-24T02:51:30.354 INFO:teuthology.orchestra.run.vm07.stdout:{"pg_ready":true,"pg_map":{"version":1526,"stamp":"2026-03-24T02:51:29.208573+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":459299,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":59,"num_write_kb":586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35,"ondisk_log_size":35,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2887556,"kb_used_data":14708,"kb_used_omap":402,"kb_used_meta":2872365,"kb_avail":280227964,"statfs":{"total":289910292480,"available":286953435136,"internally_reserved":0,"allocated":15060992,"data_stored":8063801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":411912,"internal_metadata":2941302520},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-515511927,"num_objects":-182,"num_object_clones":-4,"num_object_copies":-364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-182,"num_whiteouts":-4,"num_read":-659057,"num_read_kb":-47999777,"num_write":-648459,"num_write_kb":-19170673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-8,"num_bytes_recovered":-23294588,"num_keys_recovered":-4,"num_objects_omap":-19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.000800"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":801,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.637533+0000","last_change":"2026-03-24T02:13:56.291116+0000","last_active":"2026-03-24T02:51:24.637533+0000","last_peered":"2026-03-24T02:51:24.637533+0000","last_clean":"2026-03-24T02:51:24.637533+0000","last_became_active":"2026-03-24T02:13:54.285179+0000","last_became_peered":"2026-03-24T02:13:54.285179+0000","last_unstale":"2026-03-24T02:51:24.637533+0000","last_undegraded":"2026-03-24T02:51:24.637533+0000","last_fullsized":"2026-03-24T02:51:24.637533+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:40:35.710968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052393000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":801,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.637449+0000","last_change":"2026-03-24T02:13:56.291113+0000","last_active":"2026-03-24T02:51:24.637449+0000","last_peered":"2026-03-24T02:51:24.637449+0000","last_clean":"2026-03-24T02:51:24.637449+0000","last_became_active":"2026-03-24T02:13:54.286185+0000","last_became_peered":"2026-03-24T02:13:54.286185+0000","last_unstale":"2026-03-24T02:51:24.637449+0000","last_undegraded":"2026-03-24T02:51:24.637449+0000","last_fullsized":"2026-03-24T02:51:24.637449+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:50:07.713703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041187999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":801,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.637381+0000","last_change":"2026-03-24T02:13:56.291271+0000","last_active":"2026-03-24T02:51:24.637381+0000","last_peered":"2026-03-24T02:51:24.637381+0000","last_clean":"2026-03-24T02:51:24.637381+0000","last_became_active":"2026-03-24T02:13:54.284043+0000","last_became_peered":"2026-03-24T02:13:54.284043+0000","last_unstale":"2026-03-24T02:51:24.637381+0000","last_undegraded":"2026-03-24T02:51:24.637381+0000","last_fullsized":"2026-03-24T02:51:24.637381+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:42:40.008023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045964999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":801,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.637308+0000","last_change":"2026-03-24T02:13:56.291308+0000","last_active":"2026-03-24T02:51:24.637308+0000","last_peered":"2026-03-24T02:51:24.637308+0000","last_clean":"2026-03-24T02:51:24.637308+0000","last_became_active":"2026-03-24T02:13:54.283892+0000","last_became_peered":"2026-03-24T02:13:54.283892+0000","last_unstale":"2026-03-24T02:51:24.637308+0000","last_undegraded":"2026-03-24T02:51:24.637308+0000","last_fullsized":"2026-03-24T02:51:24.637308+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T08:17:26.341514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039982800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":804,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.633292+0000","last_change":"2026-03-24T02:13:56.288111+0000","last_active":"2026-03-24T02:51:24.633292+0000","last_peered":"2026-03-24T02:51:24.633292+0000","last_clean":"2026-03-24T02:51:24.633292+0000","last_became_active":"2026-03-24T02:13:54.283188+0000","last_became_peered":"2026-03-24T02:13:54.283188+0000","last_unstale":"2026-03-24T02:51:24.633292+0000","last_undegraded":"2026-03-24T02:51:24.633292+0000","last_fullsized":"2026-03-24T02:51:24.633292+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:58:21.144364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031652200000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":804,"reported_epoch":397,"state":"active+clean","last_fresh":"2026-03-24T02:51:26.656219+0000","last_change":"2026-03-24T02:13:56.736302+0000","last_active":"2026-03-24T02:51:26.656219+0000","last_peered":"2026-03-24T02:51:26.656219+0000","last_clean":"2026-03-24T02:51:26.656219+0000","last_became_active":"2026-03-24T02:13:54.284823+0000","last_became_peered":"2026-03-24T02:13:54.284823+0000","last_unstale":"2026-03-24T02:51:26.656219+0000","last_undegraded":"2026-03-24T02:51:26.656219+0000","last_fullsized":"2026-03-24T02:51:26.656219+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T12:50:46.136937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000322784,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":804,"reported_epoch":397,"state":"active+clean","last_fresh":"2026-03-24T02:51:26.655980+0000","last_change":"2026-03-24T02:13:56.736294+0000","last_active":"2026-03-24T02:51:26.655980+0000","last_peered":"2026-03-24T02:51:26.655980+0000","last_clean":"2026-03-24T02:51:26.655980+0000","last_became_active":"2026-03-24T02:13:54.284987+0000","last_became_peered":"2026-03-24T02:13:54.284987+0000","last_unstale":"2026-03-24T02:51:26.655980+0000","last_undegraded":"2026-03-24T02:51:26.655980+0000","last_fullsized":"2026-03-24T02:51:26.655980+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:52:45.132362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025170100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":803,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.636909+0000","last_change":"2026-03-24T02:13:56.291437+0000","last_active":"2026-03-24T02:51:24.636909+0000","last_peered":"2026-03-24T02:51:24.636909+0000","last_clean":"2026-03-24T02:51:24.636909+0000","last_became_active":"2026-03-24T02:13:54.283277+0000","last_became_peered":"2026-03-24T02:13:54.283277+0000","last_unstale":"2026-03-24T02:51:24.636909+0000","last_undegraded":"2026-03-24T02:51:24.636909+0000","last_fullsized":"2026-03-24T02:51:24.636909+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T10:25:36.882503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039834499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'32","reported_seq":845,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.636875+0000","last_change":"2026-03-24T02:13:52.693536+0000","last_active":"2026-03-24T02:51:24.636875+0000","last_peered":"2026-03-24T02:51:24.636875+0000","last_clean":"2026-03-24T02:51:24.636875+0000","last_became_active":"2026-03-24T02:13:52.693283+0000","last_became_peered":"2026-03-24T02:13:52.693283+0000","last_unstale":"2026-03-24T02:51:24.636875+0000","last_undegraded":"2026-03-24T02:51:24.636875+0000","last_fullsized":"2026-03-24T02:51:24.636875+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:51.152553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:51.152553+0000","last_clean_scrub_stamp":"2026-03-24T02:13:51.152553+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:26:45.574238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":925696,"data_stored":918560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32,"ondisk_log_size":32,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738822,"num_pgs":80,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":840972,"kb_used_data":4588,"kb_used_omap":111,"kb_used_meta":836240,"kb_avail":93530868,"statfs":{"total":96636764160,"available":95775608832,"internally_reserved":0,"allocated":4698112,"data_stored":2368043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":114282,"internal_metadata":856310166},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738821,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1509192,"kb_used_data":5040,"kb_used_omap":141,"kb_used_meta":1503986,"kb_avail":92862648,"statfs":{"total":96636764160,"available":95091351552,"internally_reserved":0,"allocated":5160960,"data_stored":2831495,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":145407,"internal_metadata":1540081665},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738824,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":537392,"kb_used_data":5080,"kb_used_omap":148,"kb_used_meta":532139,"kb_avail":93834448,"statfs":{"total":96636764160,"available":96086474752,"internally_reserved":0,"allocated":5201920,"data_stored":2864263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":152223,"internal_metadata":544910689},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-24T02:51:30.354 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-03-24T02:51:30.524 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:51:30.524 INFO:teuthology.orchestra.run.vm07.stdout:{"epoch":398,"fsid":"91ca8768-ba68-432d-bc1f-efeea617a922","created":"2026-03-24T02:13:44.473145+0000","modified":"2026-03-24T02:51:28.714982+0000","last_up_change":"2026-03-24T02:13:49.762739+0000","last_in_change":"2026-03-24T02:13:45.559913+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-24T02:13:50.810447+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-03-24T02:13:53.059072+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f02a1366-0e32-4132-9124-8b60215ba403","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6816","nonce":2403182922},{"type":"v1","addr":"192.168.123.107:6817","nonce":2403182922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6818","nonce":2403182922},{"type":"v1","addr":"192.168.123.107:6819","nonce":2403182922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6822","nonce":2403182922},{"type":"v1","addr":"192.168.123.107:6823","nonce":2403182922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6820","nonce":2403182922},{"type":"v1","addr":"192.168.123.107:6821","nonce":2403182922}]},"public_addr":"192.168.123.107:6817/2403182922","cluster_addr":"192.168.123.107:6819/2403182922","heartbeat_back_addr":"192.168.123.107:6823/2403182922","heartbeat_front_addr":"192.168.123.107:6821/2403182922","state":["exists","up"]},{"osd":1,"uuid":"c3c9032d-80f2-481b-b324-d91af15845ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6808","nonce":1372859641},{"type":"v1","addr":"192.168.123.107:6809","nonce":1372859641}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6810","nonce":1372859641},{"type":"v1","addr":"192.168.123.107:6811","nonce":1372859641}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6814","nonce":1372859641},{"type":"v1","addr":"192.168.123.107:6815","nonce":1372859641}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6812","nonce":1372859641},{"type":"v1","addr":"192.168.123.107:6813","nonce":1372859641}]},"public_addr":"192.168.123.107:6809/1372859641","cluster_addr":"192.168.123.107:6811/1372859641","heartbeat_back_addr":"192.168.123.107:6815/1372859641","heartbeat_front_addr":"192.168.123.107:6813/1372859641","state":["exists","up"]},{"osd":2,"uuid":"bca91acd-ce62-44d0-b940-2dcc1877d928","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":29,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6800","nonce":2006070968},{"type":"v1","addr":"192.168.123.107:6801","nonce":2006070968}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6802","nonce":2006070968},{"type":"v1","addr":"192.168.123.107:6803","nonce":2006070968}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6806","nonce":2006070968},{"type":"v1","addr":"192.168.123.107:6807","nonce":2006070968}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6804","nonce":2006070968},{"type":"v1","addr":"192.168.123.107:6805","nonce":2006070968}]},"public_addr":"192.168.123.107:6801/2006070968","cluster_addr":"192.168.123.107:6803/2006070968","heartbeat_back_addr":"192.168.123.107:6807/2006070968","heartbeat_front_addr":"192.168.123.107:6805/2006070968","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-24T02:13:47.664424+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-24T02:13:47.697934+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-24T02:13:47.730431+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"192.168.123.107:0/2465731803":"2026-03-24T03:51:24.590708+0000","192.168.123.107:0/243077510":"2026-03-24T03:49:32.575379+0000","192.168.123.107:0/2578257540":"2026-03-24T03:48:50.851022+0000","192.168.123.107:0/2780150297":"2026-03-24T03:47:23.949218+0000","192.168.123.107:0/1600389070":"2026-03-24T03:46:41.562392+0000","192.168.123.107:0/2992217029":"2026-03-24T03:45:47.712466+0000","192.168.123.107:0/924478062":"2026-03-24T03:45:06.506122+0000","192.168.123.107:0/3254127648":"2026-03-24T03:30:40.951517+0000","192.168.123.107:0/2674870214":"2026-03-24T03:34:22.486002+0000","192.168.123.107:0/209210464":"2026-03-24T03:30:34.528973+0000","192.168.123.107:0/2875391967":"2026-03-24T03:35:31.032652+0000","192.168.123.107:0/1578866333":"2026-03-24T03:37:15.628421+0000","192.168.123.107:0/3976351574":"2026-03-24T03:26:32.424433+0000","192.168.123.107:0/443505011":"2026-03-24T03:35:20.181703+0000","192.168.123.107:0/1416483784":"2026-03-24T03:37:50.972501+0000","192.168.123.107:0/201982731":"2026-03-24T03:27:57.456224+0000","192.168.123.107:0/1727938241":"2026-03-24T03:36:03.039658+0000","192.168.123.107:0/2947372703":"2026-03-24T03:27:50.791796+0000","192.168.123.107:0/3476342930":"2026-03-24T03:34:46.268831+0000","192.168.123.107:0/180291806":"2026-03-24T03:48:00.866821+0000","192.168.123.107:0/477443096":"2026-03-24T03:27:44.488601+0000","192.168.123.107:0/634918727":"2026-03-24T03:33:58.422641+0000","192.168.123.107:0/355631614":"2026-03-24T03:34:18.165614+0000","192.168.123.107:0/2015643568":"2026-03-24T03:35:46.536980+0000","192.168.123.107:0/267068999":"2026-03-24T03:34:02.999104+0000","192.168.123.107:0/2273346005":"2026-03-24T03:27:36.942357+0000","192.168.123.107:0/4217933265":"2026-03-24T03:27:16.916785+0000","192.168.123.107:0/817111150":"2026-03-24T03:31:18.053802+0000","192.168.123.107:0/4007454614":"2026-03-24T03:31:20.096033+0000","192.168.123.107:0/4049651436":"2026-03-24T03:24:20.859049+0000","192.168.123.107:0/1016957868":"2026-03-24T03:50:10.298882+0000","192.168.123.107:0/1993530184":"2026-03-24T03:26:48.825350+0000","192.168.123.107:0/3739979910":"2026-03-24T03:20:48.234690+0000","192.168.123.107:0/2755541629":"2026-03-24T03:32:42.428425+0000","192.168.123.107:0/1077715974":"2026-03-24T03:34:33.582072+0000","192.168.123.107:0/2007886752":"2026-03-24T03:29:27.108407+0000","192.168.123.107:0/3066612613":"2026-03-24T03:30:24.888225+0000","192.168.123.107:0/3943385180":"2026-03-24T03:16:58.948220+0000","192.168.123.107:0/3375418829":"2026-03-24T03:28:20.495777+0000","192.168.123.107:0/4235100589":"2026-03-24T03:29:27.332691+0000","192.168.123.107:0/2188795271":"2026-03-24T03:35:44.538034+0000","192.168.123.107:0/1113002042":"2026-03-24T03:17:45.618726+0000","192.168.123.107:0/4254103794":"2026-03-24T03:18:49.129832+0000","192.168.123.107:0/3025482126":"2026-03-24T03:38:28.647039+0000","192.168.123.107:0/420229945":"2026-03-24T03:31:25.478031+0000","192.168.123.107:0/2130554260":"2026-03-24T03:33:56.352293+0000","192.168.123.107:0/507632413":"2026-03-24T03:36:14.072889+0000","192.168.123.107:0/3478011426":"2026-03-24T03:19:43.834448+0000","192.168.123.107:0/1946154758":"2026-03-24T03:16:08.028157+0000","192.168.123.107:0/2261640898":"2026-03-24T03:28:08.911906+0000","192.168.123.107:0/3897076426":"2026-03-24T03:28:47.275815+0000","192.168.123.107:0/1823804729":"2026-03-24T03:26:42.407411+0000","192.168.123.107:0/1284813057":"2026-03-24T03:23:28.191833+0000","192.168.123.107:0/615776547":"2026-03-24T03:35:20.180800+0000","192.168.123.107:0/1353506773":"2026-03-24T03:21:37.596586+0000","192.168.123.107:0/2504901235":"2026-03-24T03:32:25.165586+0000","192.168.123.107:0/929055313":"2026-03-24T03:33:04.297509+0000","192.168.123.107:0/2600067232":"2026-03-24T03:35:09.622059+0000","192.168.123.107:0/405290472":"2026-03-24T03:33:52.337612+0000","192.168.123.107:0/3129127732":"2026-03-24T03:50:42.912641+0000","192.168.123.107:0/3132152272":"2026-03-24T03:34:34.556929+0000","192.168.123.107:0/2956034759":"2026-03-24T03:34:37.935135+0000","192.168.123.107:0/2483476199":"2026-03-24T03:35:18.126086+0000","192.168.123.107:0/4146803417":"2026-03-24T03:35:58.067104+0000","192.168.123.107:0/1701703093":"2026-03-24T03:26:30.431032+0000","192.168.123.107:0/1151473562":"2026-03-24T03:35:50.586296+0000","192.168.123.107:0/1684652547":"2026-03-24T03:36:17.083954+0000","192.168.123.107:0/278295179":"2026-03-24T03:22:30.291957+0000","192.168.123.107:0/2013522042":"2026-03-24T03:36:20.222673+0000","192.168.123.107:0/302594275":"2026-03-24T03:38:02.986660+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-24T02:51:31.542 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2026-03-24T02:51:31.542 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-03-24T02:51:31.623 INFO:teuthology.orchestra.run.vm07.stdout:{ 2026-03-24T02:51:31.623 INFO:teuthology.orchestra.run.vm07.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-03-24T02:51:31.623 INFO:teuthology.orchestra.run.vm07.stdout:} 2026-03-24T02:51:31.635 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2026-03-24T02:51:31.811 INFO:teuthology.orchestra.run.vm07.stderr:instructed osd(s) 0 to deep-scrub 2026-03-24T02:51:31.824 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2026-03-24T02:51:31.824 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-03-24T02:51:31.907 INFO:teuthology.orchestra.run.vm07.stdout:{ 2026-03-24T02:51:31.907 INFO:teuthology.orchestra.run.vm07.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-03-24T02:51:31.907 INFO:teuthology.orchestra.run.vm07.stdout:} 2026-03-24T02:51:31.918 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2026-03-24T02:51:32.081 INFO:teuthology.orchestra.run.vm07.stderr:instructed osd(s) 1 to deep-scrub 2026-03-24T02:51:32.096 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2026-03-24T02:51:32.096 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-03-24T02:51:32.180 INFO:teuthology.orchestra.run.vm07.stdout:{ 2026-03-24T02:51:32.180 INFO:teuthology.orchestra.run.vm07.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-03-24T02:51:32.180 INFO:teuthology.orchestra.run.vm07.stdout:} 2026-03-24T02:51:32.193 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2026-03-24T02:51:32.359 INFO:teuthology.orchestra.run.vm07.stderr:instructed osd(s) 2 to deep-scrub 2026-03-24T02:51:32.373 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-03-24T02:51:32.535 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:51:32.536 INFO:teuthology.orchestra.run.vm07.stderr:dumped all 2026-03-24T02:51:32.550 INFO:teuthology.orchestra.run.vm07.stdout:{"pg_ready":true,"pg_map":{"version":1527,"stamp":"2026-03-24T02:51:31.208799+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":459299,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":59,"num_write_kb":586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35,"ondisk_log_size":35,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2905620,"kb_used_data":14724,"kb_used_omap":404,"kb_used_meta":2890411,"kb_avail":280209900,"statfs":{"total":289910292480,"available":286934937600,"internally_reserved":0,"allocated":15077376,"data_stored":8066263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":413910,"internal_metadata":2959781674},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-515511927,"num_objects":-182,"num_object_clones":-4,"num_object_copies":-364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-182,"num_whiteouts":-4,"num_read":-659057,"num_read_kb":-47999777,"num_write":-648459,"num_write_kb":-19170673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-8,"num_bytes_recovered":-23294588,"num_keys_recovered":-4,"num_objects_omap":-19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.583650"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":805,"reported_epoch":398,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.332736+0000","last_change":"2026-03-24T02:13:56.291116+0000","last_active":"2026-03-24T02:51:29.332736+0000","last_peered":"2026-03-24T02:51:29.332736+0000","last_clean":"2026-03-24T02:51:29.332736+0000","last_became_active":"2026-03-24T02:13:54.285179+0000","last_became_peered":"2026-03-24T02:13:54.285179+0000","last_unstale":"2026-03-24T02:51:29.332736+0000","last_undegraded":"2026-03-24T02:51:29.332736+0000","last_fullsized":"2026-03-24T02:51:29.332736+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:40:35.710968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052393000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":805,"reported_epoch":398,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.332695+0000","last_change":"2026-03-24T02:13:56.291113+0000","last_active":"2026-03-24T02:51:29.332695+0000","last_peered":"2026-03-24T02:51:29.332695+0000","last_clean":"2026-03-24T02:51:29.332695+0000","last_became_active":"2026-03-24T02:13:54.286185+0000","last_became_peered":"2026-03-24T02:13:54.286185+0000","last_unstale":"2026-03-24T02:51:29.332695+0000","last_undegraded":"2026-03-24T02:51:29.332695+0000","last_fullsized":"2026-03-24T02:51:29.332695+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:50:07.713703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041187999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":805,"reported_epoch":398,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.332426+0000","last_change":"2026-03-24T02:13:56.291271+0000","last_active":"2026-03-24T02:51:29.332426+0000","last_peered":"2026-03-24T02:51:29.332426+0000","last_clean":"2026-03-24T02:51:29.332426+0000","last_became_active":"2026-03-24T02:13:54.284043+0000","last_became_peered":"2026-03-24T02:13:54.284043+0000","last_unstale":"2026-03-24T02:51:29.332426+0000","last_undegraded":"2026-03-24T02:51:29.332426+0000","last_fullsized":"2026-03-24T02:51:29.332426+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:42:40.008023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045964999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":806,"reported_epoch":398,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.332269+0000","last_change":"2026-03-24T02:13:56.291308+0000","last_active":"2026-03-24T02:51:29.332269+0000","last_peered":"2026-03-24T02:51:29.332269+0000","last_clean":"2026-03-24T02:51:29.332269+0000","last_became_active":"2026-03-24T02:13:54.283892+0000","last_became_peered":"2026-03-24T02:13:54.283892+0000","last_unstale":"2026-03-24T02:51:29.332269+0000","last_undegraded":"2026-03-24T02:51:29.332269+0000","last_fullsized":"2026-03-24T02:51:29.332269+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T08:17:26.341514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039982800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":804,"reported_epoch":396,"state":"active+clean","last_fresh":"2026-03-24T02:51:24.633292+0000","last_change":"2026-03-24T02:13:56.288111+0000","last_active":"2026-03-24T02:51:24.633292+0000","last_peered":"2026-03-24T02:51:24.633292+0000","last_clean":"2026-03-24T02:51:24.633292+0000","last_became_active":"2026-03-24T02:13:54.283188+0000","last_became_peered":"2026-03-24T02:13:54.283188+0000","last_unstale":"2026-03-24T02:51:24.633292+0000","last_undegraded":"2026-03-24T02:51:24.633292+0000","last_fullsized":"2026-03-24T02:51:24.633292+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:58:21.144364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031652200000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":804,"reported_epoch":397,"state":"active+clean","last_fresh":"2026-03-24T02:51:26.656219+0000","last_change":"2026-03-24T02:13:56.736302+0000","last_active":"2026-03-24T02:51:26.656219+0000","last_peered":"2026-03-24T02:51:26.656219+0000","last_clean":"2026-03-24T02:51:26.656219+0000","last_became_active":"2026-03-24T02:13:54.284823+0000","last_became_peered":"2026-03-24T02:13:54.284823+0000","last_unstale":"2026-03-24T02:51:26.656219+0000","last_undegraded":"2026-03-24T02:51:26.656219+0000","last_fullsized":"2026-03-24T02:51:26.656219+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T12:50:46.136937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000322784,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":804,"reported_epoch":397,"state":"active+clean","last_fresh":"2026-03-24T02:51:26.655980+0000","last_change":"2026-03-24T02:13:56.736294+0000","last_active":"2026-03-24T02:51:26.655980+0000","last_peered":"2026-03-24T02:51:26.655980+0000","last_clean":"2026-03-24T02:51:26.655980+0000","last_became_active":"2026-03-24T02:13:54.284987+0000","last_became_peered":"2026-03-24T02:13:54.284987+0000","last_unstale":"2026-03-24T02:51:26.655980+0000","last_undegraded":"2026-03-24T02:51:26.655980+0000","last_fullsized":"2026-03-24T02:51:26.655980+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:52:45.132362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025170100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":807,"reported_epoch":398,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.331836+0000","last_change":"2026-03-24T02:13:56.291437+0000","last_active":"2026-03-24T02:51:29.331836+0000","last_peered":"2026-03-24T02:51:29.331836+0000","last_clean":"2026-03-24T02:51:29.331836+0000","last_became_active":"2026-03-24T02:13:54.283277+0000","last_became_peered":"2026-03-24T02:13:54.283277+0000","last_unstale":"2026-03-24T02:51:29.331836+0000","last_undegraded":"2026-03-24T02:51:29.331836+0000","last_fullsized":"2026-03-24T02:51:29.331836+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:53.270179+0000","last_clean_scrub_stamp":"2026-03-24T02:13:53.270179+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T10:25:36.882503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039834499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'32","reported_seq":849,"reported_epoch":398,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.326772+0000","last_change":"2026-03-24T02:13:52.693536+0000","last_active":"2026-03-24T02:51:29.326772+0000","last_peered":"2026-03-24T02:51:29.326772+0000","last_clean":"2026-03-24T02:51:29.326772+0000","last_became_active":"2026-03-24T02:13:52.693283+0000","last_became_peered":"2026-03-24T02:13:52.693283+0000","last_unstale":"2026-03-24T02:51:29.326772+0000","last_undegraded":"2026-03-24T02:51:29.326772+0000","last_fullsized":"2026-03-24T02:51:29.326772+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:51.152553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:51.152553+0000","last_clean_scrub_stamp":"2026-03-24T02:13:51.152553+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:26:45.574238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":925696,"data_stored":918560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32,"ondisk_log_size":32,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738822,"num_pgs":80,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":840972,"kb_used_data":4588,"kb_used_omap":111,"kb_used_meta":836240,"kb_avail":93530868,"statfs":{"total":96636764160,"available":95775608832,"internally_reserved":0,"allocated":4698112,"data_stored":2368043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":114282,"internal_metadata":856310166},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738822,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1527256,"kb_used_data":5056,"kb_used_omap":143,"kb_used_meta":1522032,"kb_avail":92844584,"statfs":{"total":96636764160,"available":95072854016,"internally_reserved":0,"allocated":5177344,"data_stored":2833957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":147405,"internal_metadata":1558560819},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738824,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":537392,"kb_used_data":5080,"kb_used_omap":148,"kb_used_meta":532139,"kb_avail":93834448,"statfs":{"total":96636764160,"available":96086474752,"internally_reserved":0,"allocated":5201920,"data_stored":2864263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":152223,"internal_metadata":544910689},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-24T02:51:32.672 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-03-24T02:13:53.270179+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=53, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=30, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:32.672 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-03-24T02:13:53.270179+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=53, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=30, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:32.672 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-03-24T02:13:53.270179+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=53, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=30, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:32.672 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-03-24T02:13:53.270179+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=53, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=30, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:32.672 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-03-24T02:13:53.270179+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=53, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=30, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:32.672 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-03-24T02:13:53.270179+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=53, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=30, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:32.672 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-03-24T02:13:53.270179+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=53, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=30, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:32.672 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-03-24T02:13:53.270179+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=53, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=30, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:32.672 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-03-24T02:13:51.152553+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=51, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=30, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:32.672 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-03-24T02:51:52.673 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-03-24T02:51:52.838 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-24T02:51:52.839 INFO:teuthology.orchestra.run.vm07.stderr:dumped all 2026-03-24T02:51:52.853 INFO:teuthology.orchestra.run.vm07.stdout:{"pg_ready":true,"pg_map":{"version":1537,"stamp":"2026-03-24T02:51:51.210947+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":459299,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":59,"num_write_kb":586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2941672,"kb_used_data":14712,"kb_used_omap":407,"kb_used_meta":2926504,"kb_avail":280173848,"statfs":{"total":289910292480,"available":286898020352,"internally_reserved":0,"allocated":15065088,"data_stored":8042572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":416913,"internal_metadata":2996740975},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001237"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":813,"reported_epoch":398,"state":"active+clean","last_fresh":"2026-03-24T02:51:36.851743+0000","last_change":"2026-03-24T02:51:36.851743+0000","last_active":"2026-03-24T02:51:36.851743+0000","last_peered":"2026-03-24T02:51:36.851743+0000","last_clean":"2026-03-24T02:51:36.851743+0000","last_became_active":"2026-03-24T02:13:54.285179+0000","last_became_peered":"2026-03-24T02:13:54.285179+0000","last_unstale":"2026-03-24T02:51:36.851743+0000","last_undegraded":"2026-03-24T02:51:36.851743+0000","last_fullsized":"2026-03-24T02:51:36.851743+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:51:36.851702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:51:36.851702+0000","last_clean_scrub_stamp":"2026-03-24T02:51:36.851702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T12:30:13.645730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052393000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":813,"reported_epoch":398,"state":"active+clean","last_fresh":"2026-03-24T02:51:37.804524+0000","last_change":"2026-03-24T02:51:37.804524+0000","last_active":"2026-03-24T02:51:37.804524+0000","last_peered":"2026-03-24T02:51:37.804524+0000","last_clean":"2026-03-24T02:51:37.804524+0000","last_became_active":"2026-03-24T02:13:54.286185+0000","last_became_peered":"2026-03-24T02:13:54.286185+0000","last_unstale":"2026-03-24T02:51:37.804524+0000","last_undegraded":"2026-03-24T02:51:37.804524+0000","last_fullsized":"2026-03-24T02:51:37.804524+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:51:37.804477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:51:37.804477+0000","last_clean_scrub_stamp":"2026-03-24T02:51:37.804477+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T13:14:41.054210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041187999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":814,"reported_epoch":398,"state":"active+clean","last_fresh":"2026-03-24T02:51:35.829259+0000","last_change":"2026-03-24T02:51:35.829220+0000","last_active":"2026-03-24T02:51:35.829259+0000","last_peered":"2026-03-24T02:51:35.829259+0000","last_clean":"2026-03-24T02:51:35.829259+0000","last_became_active":"2026-03-24T02:13:54.284043+0000","last_became_peered":"2026-03-24T02:13:54.284043+0000","last_unstale":"2026-03-24T02:51:35.829259+0000","last_undegraded":"2026-03-24T02:51:35.829259+0000","last_fullsized":"2026-03-24T02:51:35.829259+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:51:35.829178+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:51:35.829178+0000","last_clean_scrub_stamp":"2026-03-24T02:51:35.829178+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:43:47.040951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045964999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":814,"reported_epoch":398,"state":"active+clean","last_fresh":"2026-03-24T02:51:33.798932+0000","last_change":"2026-03-24T02:51:33.798932+0000","last_active":"2026-03-24T02:51:33.798932+0000","last_peered":"2026-03-24T02:51:33.798932+0000","last_clean":"2026-03-24T02:51:33.798932+0000","last_became_active":"2026-03-24T02:13:54.283892+0000","last_became_peered":"2026-03-24T02:13:54.283892+0000","last_unstale":"2026-03-24T02:51:33.798932+0000","last_undegraded":"2026-03-24T02:51:33.798932+0000","last_fullsized":"2026-03-24T02:51:33.798932+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:51:33.798884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:51:33.798884+0000","last_clean_scrub_stamp":"2026-03-24T02:51:33.798884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T07:13:06.830106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039982800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"398'3","reported_seq":817,"reported_epoch":398,"state":"active+clean","last_fresh":"2026-03-24T02:51:32.163258+0000","last_change":"2026-03-24T02:51:32.163258+0000","last_active":"2026-03-24T02:51:32.163258+0000","last_peered":"2026-03-24T02:51:32.163258+0000","last_clean":"2026-03-24T02:51:32.163258+0000","last_became_active":"2026-03-24T02:13:54.283188+0000","last_became_peered":"2026-03-24T02:13:54.283188+0000","last_unstale":"2026-03-24T02:51:32.163258+0000","last_undegraded":"2026-03-24T02:51:32.163258+0000","last_fullsized":"2026-03-24T02:51:32.163258+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"398'3","last_scrub_stamp":"2026-03-24T02:51:32.163227+0000","last_deep_scrub":"398'3","last_deep_scrub_stamp":"2026-03-24T02:51:32.163227+0000","last_clean_scrub_stamp":"2026-03-24T02:51:32.163227+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:09:54.353795+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0.00031652200000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":814,"reported_epoch":398,"state":"active+clean","last_fresh":"2026-03-24T02:51:33.970958+0000","last_change":"2026-03-24T02:51:33.970958+0000","last_active":"2026-03-24T02:51:33.970958+0000","last_peered":"2026-03-24T02:51:33.970958+0000","last_clean":"2026-03-24T02:51:33.970958+0000","last_became_active":"2026-03-24T02:13:54.284823+0000","last_became_peered":"2026-03-24T02:13:54.284823+0000","last_unstale":"2026-03-24T02:51:33.970958+0000","last_undegraded":"2026-03-24T02:51:33.970958+0000","last_fullsized":"2026-03-24T02:51:33.970958+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:51:33.970916+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:51:33.970916+0000","last_clean_scrub_stamp":"2026-03-24T02:51:33.970916+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:03:43.080054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000322784,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":814,"reported_epoch":398,"state":"active+clean","last_fresh":"2026-03-24T02:51:32.979119+0000","last_change":"2026-03-24T02:51:32.979119+0000","last_active":"2026-03-24T02:51:32.979119+0000","last_peered":"2026-03-24T02:51:32.979119+0000","last_clean":"2026-03-24T02:51:32.979119+0000","last_became_active":"2026-03-24T02:13:54.284987+0000","last_became_peered":"2026-03-24T02:13:54.284987+0000","last_unstale":"2026-03-24T02:51:32.979119+0000","last_undegraded":"2026-03-24T02:51:32.979119+0000","last_fullsized":"2026-03-24T02:51:32.979119+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:51:32.979079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:51:32.979079+0000","last_clean_scrub_stamp":"2026-03-24T02:51:32.979079+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:16:50.932874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025170100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"398'2","reported_seq":816,"reported_epoch":398,"state":"active+clean","last_fresh":"2026-03-24T02:51:34.830815+0000","last_change":"2026-03-24T02:51:34.830815+0000","last_active":"2026-03-24T02:51:34.830815+0000","last_peered":"2026-03-24T02:51:34.830815+0000","last_clean":"2026-03-24T02:51:34.830815+0000","last_became_active":"2026-03-24T02:13:54.283277+0000","last_became_peered":"2026-03-24T02:13:54.283277+0000","last_unstale":"2026-03-24T02:51:34.830815+0000","last_undegraded":"2026-03-24T02:51:34.830815+0000","last_fullsized":"2026-03-24T02:51:34.830815+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"398'2","last_scrub_stamp":"2026-03-24T02:51:34.830774+0000","last_deep_scrub":"398'2","last_deep_scrub_stamp":"2026-03-24T02:51:34.830774+0000","last_clean_scrub_stamp":"2026-03-24T02:51:34.830774+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T10:40:26.064321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039834499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"398'34","reported_seq":859,"reported_epoch":398,"state":"active+clean","last_fresh":"2026-03-24T02:51:32.809540+0000","last_change":"2026-03-24T02:51:32.809540+0000","last_active":"2026-03-24T02:51:32.809540+0000","last_peered":"2026-03-24T02:51:32.809540+0000","last_clean":"2026-03-24T02:51:32.809540+0000","last_became_active":"2026-03-24T02:13:52.693283+0000","last_became_peered":"2026-03-24T02:13:52.693283+0000","last_unstale":"2026-03-24T02:51:32.809540+0000","last_undegraded":"2026-03-24T02:51:32.809540+0000","last_fullsized":"2026-03-24T02:51:32.809540+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"398'34","last_scrub_stamp":"2026-03-24T02:51:32.809499+0000","last_deep_scrub":"398'34","last_deep_scrub_stamp":"2026-03-24T02:51:32.809499+0000","last_clean_scrub_stamp":"2026-03-24T02:51:32.809499+0000","objects_scrubbed":2,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T03:47:15.433736+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":925696,"data_stored":918560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":34,"ondisk_log_size":34,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738826,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":840968,"kb_used_data":4600,"kb_used_omap":113,"kb_used_meta":836238,"kb_avail":93530872,"statfs":{"total":96636764160,"available":95775612928,"internally_reserved":0,"allocated":4710400,"data_stored":2374658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":116289,"internal_metadata":856308159},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738826,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1527248,"kb_used_data":5056,"kb_used_omap":143,"kb_used_meta":1522032,"kb_avail":92844592,"statfs":{"total":96636764160,"available":95072862208,"internally_reserved":0,"allocated":5177344,"data_stored":2833957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":147405,"internal_metadata":1558560819},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738828,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":573456,"kb_used_data":5056,"kb_used_omap":149,"kb_used_meta":568234,"kb_avail":93798384,"statfs":{"total":96636764160,"available":96049545216,"internally_reserved":0,"allocated":5177344,"data_stored":2833957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":153219,"internal_metadata":581871997},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-24T02:51:52.854 DEBUG:teuthology.orchestra.run.vm07:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2026-03-24T02:51:53.036 INFO:teuthology.misc:Shutting down mds daemons... 2026-03-24T02:51:53.036 INFO:teuthology.misc:Shutting down osd daemons... 2026-03-24T02:51:53.036 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2026-03-24T02:51:53.036 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:51:53.102 INFO:tasks.ceph.cluster2.osd.0:Stopped 2026-03-24T02:51:53.103 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2026-03-24T02:51:53.103 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:51:53.179 INFO:tasks.ceph.cluster2.osd.1:Stopped 2026-03-24T02:51:53.179 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2026-03-24T02:51:53.179 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:51:53.270 INFO:tasks.ceph.cluster2.osd.2:Stopped 2026-03-24T02:51:53.271 INFO:teuthology.misc:Shutting down mgr daemons... 2026-03-24T02:51:53.271 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2026-03-24T02:51:53.271 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:51:53.301 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2026-03-24T02:51:53.301 INFO:teuthology.misc:Shutting down mon daemons... 2026-03-24T02:51:53.301 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2026-03-24T02:51:53.301 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:51:53.314 INFO:tasks.ceph.cluster2.mon.a:Stopped 2026-03-24T02:51:53.314 INFO:tasks.ceph:Checking cluster log for badness... 2026-03-24T02:51:53.315 DEBUG:teuthology.orchestra.run.vm07:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-03-24T02:51:53.370 INFO:teuthology.orchestra.run.vm07.stdout:2026-03-24T02:47:32.483601+0000 mon.a (mon.0) 1455 : cluster [ERR] Health check failed: mon a is very low on available space (MON_DISK_CRIT) 2026-03-24T02:51:53.370 WARNING:tasks.ceph:Found errors (ERR|WRN|SEC) in cluster log 2026-03-24T02:51:53.371 DEBUG:teuthology.orchestra.run.vm07:> sudo egrep '\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-03-24T02:51:53.424 DEBUG:teuthology.orchestra.run.vm07:> sudo egrep '\[ERR\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-03-24T02:51:53.475 INFO:teuthology.orchestra.run.vm07.stdout:2026-03-24T02:47:32.483601+0000 mon.a (mon.0) 1455 : cluster [ERR] Health check failed: mon a is very low on available space (MON_DISK_CRIT) 2026-03-24T02:51:53.475 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@vm07.local 2026-03-24T02:51:53.476 DEBUG:teuthology.orchestra.run.vm07:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2026-03-24T02:51:53.853 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@vm07.local 2026-03-24T02:51:53.853 DEBUG:teuthology.orchestra.run.vm07:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2026-03-24T02:51:54.148 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@vm07.local 2026-03-24T02:51:54.149 DEBUG:teuthology.orchestra.run.vm07:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2026-03-24T02:51:54.425 INFO:tasks.ceph:Archiving mon data... 2026-03-24T02:51:54.426 DEBUG:teuthology.misc:Transferring archived files from vm07:/var/lib/ceph/mon/cluster2-a to /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3539/data/cluster2.mon.a.tgz 2026-03-24T02:51:54.427 DEBUG:teuthology.orchestra.run.vm07:> mktemp 2026-03-24T02:51:54.430 INFO:teuthology.orchestra.run.vm07.stdout:/tmp/tmp.YwBb9eKra6 2026-03-24T02:51:54.430 DEBUG:teuthology.orchestra.run.vm07:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.YwBb9eKra6 2026-03-24T02:51:54.546 DEBUG:teuthology.orchestra.run.vm07:> sudo chmod 0666 /tmp/tmp.YwBb9eKra6 2026-03-24T02:51:54.606 DEBUG:teuthology.orchestra.remote:vm07:/tmp/tmp.YwBb9eKra6 is 486KB 2026-03-24T02:51:54.663 DEBUG:teuthology.orchestra.run.vm07:> rm -fr /tmp/tmp.YwBb9eKra6 2026-03-24T02:51:54.666 INFO:tasks.ceph:Cleaning ceph cluster... 2026-03-24T02:51:54.666 DEBUG:teuthology.orchestra.run.vm06:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-03-24T02:51:54.668 DEBUG:teuthology.orchestra.run.vm07:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-03-24T02:51:54.719 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-03-24T02:51:54.722 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-03-24T02:51:54.722 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-03-24T02:51:54.907 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:51:54.907 INFO:teuthology.orchestra.run.vm06.stderr:dumped all 2026-03-24T02:51:54.923 INFO:teuthology.orchestra.run.vm06.stdout:{"pg_ready":true,"pg_map":{"version":1487,"stamp":"2026-03-24T02:51:53.205429+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":459299,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":59,"num_write_kb":586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35,"ondisk_log_size":35,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3417936,"kb_used_data":9024,"kb_used_omap":3056,"kb_used_meta":3405775,"kb_avail":279697584,"statfs":{"total":289910292480,"available":286410326016,"internally_reserved":0,"allocated":9240576,"data_stored":5303182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3129410,"internal_metadata":3487514558},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001586"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":679,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.664240+0000","last_change":"2026-03-24T02:13:32.410665+0000","last_active":"2026-03-24T02:51:27.664240+0000","last_peered":"2026-03-24T02:51:27.664240+0000","last_clean":"2026-03-24T02:51:27.664240+0000","last_became_active":"2026-03-24T02:13:30.350133+0000","last_became_peered":"2026-03-24T02:13:30.350133+0000","last_unstale":"2026-03-24T02:51:27.664240+0000","last_undegraded":"2026-03-24T02:51:27.664240+0000","last_fullsized":"2026-03-24T02:51:27.664240+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:51:40.295988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035299000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":679,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662671+0000","last_change":"2026-03-24T02:13:32.410938+0000","last_active":"2026-03-24T02:51:27.662671+0000","last_peered":"2026-03-24T02:51:27.662671+0000","last_clean":"2026-03-24T02:51:27.662671+0000","last_became_active":"2026-03-24T02:13:30.349481+0000","last_became_peered":"2026-03-24T02:13:30.349481+0000","last_unstale":"2026-03-24T02:51:27.662671+0000","last_undegraded":"2026-03-24T02:51:27.662671+0000","last_fullsized":"2026-03-24T02:51:27.662671+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:35:42.157836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055518799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":680,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662534+0000","last_change":"2026-03-24T02:13:32.410738+0000","last_active":"2026-03-24T02:51:27.662534+0000","last_peered":"2026-03-24T02:51:27.662534+0000","last_clean":"2026-03-24T02:51:27.662534+0000","last_became_active":"2026-03-24T02:13:30.348547+0000","last_became_peered":"2026-03-24T02:13:30.348547+0000","last_unstale":"2026-03-24T02:51:27.662534+0000","last_undegraded":"2026-03-24T02:51:27.662534+0000","last_fullsized":"2026-03-24T02:51:27.662534+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:52:23.166796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033254200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":680,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662388+0000","last_change":"2026-03-24T02:13:32.411084+0000","last_active":"2026-03-24T02:51:27.662388+0000","last_peered":"2026-03-24T02:51:27.662388+0000","last_clean":"2026-03-24T02:51:27.662388+0000","last_became_active":"2026-03-24T02:13:30.349564+0000","last_became_peered":"2026-03-24T02:13:30.349564+0000","last_unstale":"2026-03-24T02:51:27.662388+0000","last_undegraded":"2026-03-24T02:51:27.662388+0000","last_fullsized":"2026-03-24T02:51:27.662388+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:04:43.950912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059685599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":681,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.300912+0000","last_change":"2026-03-24T02:13:32.409322+0000","last_active":"2026-03-24T02:51:29.300912+0000","last_peered":"2026-03-24T02:51:29.300912+0000","last_clean":"2026-03-24T02:51:29.300912+0000","last_became_active":"2026-03-24T02:13:30.350751+0000","last_became_peered":"2026-03-24T02:13:30.350751+0000","last_unstale":"2026-03-24T02:51:29.300912+0000","last_undegraded":"2026-03-24T02:51:29.300912+0000","last_fullsized":"2026-03-24T02:51:29.300912+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:58:18.488299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082885000000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":680,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.303000+0000","last_change":"2026-03-24T02:13:32.406625+0000","last_active":"2026-03-24T02:51:29.303000+0000","last_peered":"2026-03-24T02:51:29.303000+0000","last_clean":"2026-03-24T02:51:29.303000+0000","last_became_active":"2026-03-24T02:13:30.348773+0000","last_became_peered":"2026-03-24T02:13:30.348773+0000","last_unstale":"2026-03-24T02:51:29.303000+0000","last_undegraded":"2026-03-24T02:51:29.303000+0000","last_fullsized":"2026-03-24T02:51:29.303000+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T08:00:33.758016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00018693,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":681,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.300907+0000","last_change":"2026-03-24T02:13:32.406690+0000","last_active":"2026-03-24T02:51:29.300907+0000","last_peered":"2026-03-24T02:51:29.300907+0000","last_clean":"2026-03-24T02:51:29.300907+0000","last_became_active":"2026-03-24T02:13:30.348538+0000","last_became_peered":"2026-03-24T02:13:30.348538+0000","last_unstale":"2026-03-24T02:51:29.300907+0000","last_undegraded":"2026-03-24T02:51:29.300907+0000","last_fullsized":"2026-03-24T02:51:29.300907+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T13:03:07.820972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023985799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":681,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662285+0000","last_change":"2026-03-24T02:13:32.411138+0000","last_active":"2026-03-24T02:51:27.662285+0000","last_peered":"2026-03-24T02:51:27.662285+0000","last_clean":"2026-03-24T02:51:27.662285+0000","last_became_active":"2026-03-24T02:13:30.348335+0000","last_became_peered":"2026-03-24T02:13:30.348335+0000","last_unstale":"2026-03-24T02:51:27.662285+0000","last_undegraded":"2026-03-24T02:51:27.662285+0000","last_fullsized":"2026-03-24T02:51:27.662285+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:37:39.995117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058244000000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'32","reported_seq":726,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662254+0000","last_change":"2026-03-24T02:13:27.310722+0000","last_active":"2026-03-24T02:51:27.662254+0000","last_peered":"2026-03-24T02:51:27.662254+0000","last_clean":"2026-03-24T02:51:27.662254+0000","last_became_active":"2026-03-24T02:13:27.310550+0000","last_became_peered":"2026-03-24T02:13:27.310550+0000","last_unstale":"2026-03-24T02:51:27.662254+0000","last_undegraded":"2026-03-24T02:51:27.662254+0000","last_fullsized":"2026-03-24T02:51:27.662254+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:25.316656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:25.316656+0000","last_clean_scrub_stamp":"2026-03-24T02:13:25.316656+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:46:01.042093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":925696,"data_stored":918560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32,"ondisk_log_size":32,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738832,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1055592,"kb_used_data":2704,"kb_used_omap":1163,"kb_used_meta":1051700,"kb_avail":93316248,"statfs":{"total":96636764160,"available":95555837952,"internally_reserved":0,"allocated":2768896,"data_stored":1461528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1191587,"internal_metadata":1076941149},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738833,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1272888,"kb_used_data":3160,"kb_used_omap":660,"kb_used_meta":1269035,"kb_avail":93098952,"statfs":{"total":96636764160,"available":95333326848,"internally_reserved":0,"allocated":3235840,"data_stored":1920827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":676226,"internal_metadata":1299492478},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738831,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1089456,"kb_used_data":3160,"kb_used_omap":1232,"kb_used_meta":1085039,"kb_avail":93282384,"statfs":{"total":96636764160,"available":95521161216,"internally_reserved":0,"allocated":3235840,"data_stored":1920827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1261597,"internal_metadata":1111080931},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-24T02:51:54.924 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-03-24T02:51:55.109 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:51:55.109 INFO:teuthology.orchestra.run.vm06.stderr:dumped all 2026-03-24T02:51:55.123 INFO:teuthology.orchestra.run.vm06.stdout:{"pg_ready":true,"pg_map":{"version":1487,"stamp":"2026-03-24T02:51:53.205429+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":459299,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":59,"num_write_kb":586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35,"ondisk_log_size":35,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3417936,"kb_used_data":9024,"kb_used_omap":3056,"kb_used_meta":3405775,"kb_avail":279697584,"statfs":{"total":289910292480,"available":286410326016,"internally_reserved":0,"allocated":9240576,"data_stored":5303182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3129410,"internal_metadata":3487514558},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001586"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":679,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.664240+0000","last_change":"2026-03-24T02:13:32.410665+0000","last_active":"2026-03-24T02:51:27.664240+0000","last_peered":"2026-03-24T02:51:27.664240+0000","last_clean":"2026-03-24T02:51:27.664240+0000","last_became_active":"2026-03-24T02:13:30.350133+0000","last_became_peered":"2026-03-24T02:13:30.350133+0000","last_unstale":"2026-03-24T02:51:27.664240+0000","last_undegraded":"2026-03-24T02:51:27.664240+0000","last_fullsized":"2026-03-24T02:51:27.664240+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:51:40.295988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035299000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":679,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662671+0000","last_change":"2026-03-24T02:13:32.410938+0000","last_active":"2026-03-24T02:51:27.662671+0000","last_peered":"2026-03-24T02:51:27.662671+0000","last_clean":"2026-03-24T02:51:27.662671+0000","last_became_active":"2026-03-24T02:13:30.349481+0000","last_became_peered":"2026-03-24T02:13:30.349481+0000","last_unstale":"2026-03-24T02:51:27.662671+0000","last_undegraded":"2026-03-24T02:51:27.662671+0000","last_fullsized":"2026-03-24T02:51:27.662671+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:35:42.157836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055518799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":680,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662534+0000","last_change":"2026-03-24T02:13:32.410738+0000","last_active":"2026-03-24T02:51:27.662534+0000","last_peered":"2026-03-24T02:51:27.662534+0000","last_clean":"2026-03-24T02:51:27.662534+0000","last_became_active":"2026-03-24T02:13:30.348547+0000","last_became_peered":"2026-03-24T02:13:30.348547+0000","last_unstale":"2026-03-24T02:51:27.662534+0000","last_undegraded":"2026-03-24T02:51:27.662534+0000","last_fullsized":"2026-03-24T02:51:27.662534+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:52:23.166796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033254200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":680,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662388+0000","last_change":"2026-03-24T02:13:32.411084+0000","last_active":"2026-03-24T02:51:27.662388+0000","last_peered":"2026-03-24T02:51:27.662388+0000","last_clean":"2026-03-24T02:51:27.662388+0000","last_became_active":"2026-03-24T02:13:30.349564+0000","last_became_peered":"2026-03-24T02:13:30.349564+0000","last_unstale":"2026-03-24T02:51:27.662388+0000","last_undegraded":"2026-03-24T02:51:27.662388+0000","last_fullsized":"2026-03-24T02:51:27.662388+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:04:43.950912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059685599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":681,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.300912+0000","last_change":"2026-03-24T02:13:32.409322+0000","last_active":"2026-03-24T02:51:29.300912+0000","last_peered":"2026-03-24T02:51:29.300912+0000","last_clean":"2026-03-24T02:51:29.300912+0000","last_became_active":"2026-03-24T02:13:30.350751+0000","last_became_peered":"2026-03-24T02:13:30.350751+0000","last_unstale":"2026-03-24T02:51:29.300912+0000","last_undegraded":"2026-03-24T02:51:29.300912+0000","last_fullsized":"2026-03-24T02:51:29.300912+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:58:18.488299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082885000000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":680,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.303000+0000","last_change":"2026-03-24T02:13:32.406625+0000","last_active":"2026-03-24T02:51:29.303000+0000","last_peered":"2026-03-24T02:51:29.303000+0000","last_clean":"2026-03-24T02:51:29.303000+0000","last_became_active":"2026-03-24T02:13:30.348773+0000","last_became_peered":"2026-03-24T02:13:30.348773+0000","last_unstale":"2026-03-24T02:51:29.303000+0000","last_undegraded":"2026-03-24T02:51:29.303000+0000","last_fullsized":"2026-03-24T02:51:29.303000+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T08:00:33.758016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00018693,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":681,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.300907+0000","last_change":"2026-03-24T02:13:32.406690+0000","last_active":"2026-03-24T02:51:29.300907+0000","last_peered":"2026-03-24T02:51:29.300907+0000","last_clean":"2026-03-24T02:51:29.300907+0000","last_became_active":"2026-03-24T02:13:30.348538+0000","last_became_peered":"2026-03-24T02:13:30.348538+0000","last_unstale":"2026-03-24T02:51:29.300907+0000","last_undegraded":"2026-03-24T02:51:29.300907+0000","last_fullsized":"2026-03-24T02:51:29.300907+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T13:03:07.820972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023985799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":681,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662285+0000","last_change":"2026-03-24T02:13:32.411138+0000","last_active":"2026-03-24T02:51:27.662285+0000","last_peered":"2026-03-24T02:51:27.662285+0000","last_clean":"2026-03-24T02:51:27.662285+0000","last_became_active":"2026-03-24T02:13:30.348335+0000","last_became_peered":"2026-03-24T02:13:30.348335+0000","last_unstale":"2026-03-24T02:51:27.662285+0000","last_undegraded":"2026-03-24T02:51:27.662285+0000","last_fullsized":"2026-03-24T02:51:27.662285+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:37:39.995117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058244000000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'32","reported_seq":726,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662254+0000","last_change":"2026-03-24T02:13:27.310722+0000","last_active":"2026-03-24T02:51:27.662254+0000","last_peered":"2026-03-24T02:51:27.662254+0000","last_clean":"2026-03-24T02:51:27.662254+0000","last_became_active":"2026-03-24T02:13:27.310550+0000","last_became_peered":"2026-03-24T02:13:27.310550+0000","last_unstale":"2026-03-24T02:51:27.662254+0000","last_undegraded":"2026-03-24T02:51:27.662254+0000","last_fullsized":"2026-03-24T02:51:27.662254+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:25.316656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:25.316656+0000","last_clean_scrub_stamp":"2026-03-24T02:13:25.316656+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:46:01.042093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":925696,"data_stored":918560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32,"ondisk_log_size":32,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738832,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1055592,"kb_used_data":2704,"kb_used_omap":1163,"kb_used_meta":1051700,"kb_avail":93316248,"statfs":{"total":96636764160,"available":95555837952,"internally_reserved":0,"allocated":2768896,"data_stored":1461528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1191587,"internal_metadata":1076941149},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738833,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1272888,"kb_used_data":3160,"kb_used_omap":660,"kb_used_meta":1269035,"kb_avail":93098952,"statfs":{"total":96636764160,"available":95333326848,"internally_reserved":0,"allocated":3235840,"data_stored":1920827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":676226,"internal_metadata":1299492478},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738831,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1089456,"kb_used_data":3160,"kb_used_omap":1232,"kb_used_meta":1085039,"kb_avail":93282384,"statfs":{"total":96636764160,"available":95521161216,"internally_reserved":0,"allocated":3235840,"data_stored":1920827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1261597,"internal_metadata":1111080931},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-24T02:51:55.123 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-03-24T02:51:55.123 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-03-24T02:51:55.284 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:51:55.285 INFO:teuthology.orchestra.run.vm06.stderr:dumped all 2026-03-24T02:51:55.299 INFO:teuthology.orchestra.run.vm06.stdout:{"pg_ready":true,"pg_map":{"version":1488,"stamp":"2026-03-24T02:51:55.205647+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":459299,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":59,"num_write_kb":586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35,"ondisk_log_size":35,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3417936,"kb_used_data":9024,"kb_used_omap":3056,"kb_used_meta":3405775,"kb_avail":279697584,"statfs":{"total":289910292480,"available":286410326016,"internally_reserved":0,"allocated":9240576,"data_stored":5303182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3129410,"internal_metadata":3487514558},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001520"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":679,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.664240+0000","last_change":"2026-03-24T02:13:32.410665+0000","last_active":"2026-03-24T02:51:27.664240+0000","last_peered":"2026-03-24T02:51:27.664240+0000","last_clean":"2026-03-24T02:51:27.664240+0000","last_became_active":"2026-03-24T02:13:30.350133+0000","last_became_peered":"2026-03-24T02:13:30.350133+0000","last_unstale":"2026-03-24T02:51:27.664240+0000","last_undegraded":"2026-03-24T02:51:27.664240+0000","last_fullsized":"2026-03-24T02:51:27.664240+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:51:40.295988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035299000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":679,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662671+0000","last_change":"2026-03-24T02:13:32.410938+0000","last_active":"2026-03-24T02:51:27.662671+0000","last_peered":"2026-03-24T02:51:27.662671+0000","last_clean":"2026-03-24T02:51:27.662671+0000","last_became_active":"2026-03-24T02:13:30.349481+0000","last_became_peered":"2026-03-24T02:13:30.349481+0000","last_unstale":"2026-03-24T02:51:27.662671+0000","last_undegraded":"2026-03-24T02:51:27.662671+0000","last_fullsized":"2026-03-24T02:51:27.662671+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:35:42.157836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055518799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":680,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662534+0000","last_change":"2026-03-24T02:13:32.410738+0000","last_active":"2026-03-24T02:51:27.662534+0000","last_peered":"2026-03-24T02:51:27.662534+0000","last_clean":"2026-03-24T02:51:27.662534+0000","last_became_active":"2026-03-24T02:13:30.348547+0000","last_became_peered":"2026-03-24T02:13:30.348547+0000","last_unstale":"2026-03-24T02:51:27.662534+0000","last_undegraded":"2026-03-24T02:51:27.662534+0000","last_fullsized":"2026-03-24T02:51:27.662534+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:52:23.166796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033254200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":680,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662388+0000","last_change":"2026-03-24T02:13:32.411084+0000","last_active":"2026-03-24T02:51:27.662388+0000","last_peered":"2026-03-24T02:51:27.662388+0000","last_clean":"2026-03-24T02:51:27.662388+0000","last_became_active":"2026-03-24T02:13:30.349564+0000","last_became_peered":"2026-03-24T02:13:30.349564+0000","last_unstale":"2026-03-24T02:51:27.662388+0000","last_undegraded":"2026-03-24T02:51:27.662388+0000","last_fullsized":"2026-03-24T02:51:27.662388+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:04:43.950912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059685599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":681,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.300912+0000","last_change":"2026-03-24T02:13:32.409322+0000","last_active":"2026-03-24T02:51:29.300912+0000","last_peered":"2026-03-24T02:51:29.300912+0000","last_clean":"2026-03-24T02:51:29.300912+0000","last_became_active":"2026-03-24T02:13:30.350751+0000","last_became_peered":"2026-03-24T02:13:30.350751+0000","last_unstale":"2026-03-24T02:51:29.300912+0000","last_undegraded":"2026-03-24T02:51:29.300912+0000","last_fullsized":"2026-03-24T02:51:29.300912+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:58:18.488299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082885000000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":680,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.303000+0000","last_change":"2026-03-24T02:13:32.406625+0000","last_active":"2026-03-24T02:51:29.303000+0000","last_peered":"2026-03-24T02:51:29.303000+0000","last_clean":"2026-03-24T02:51:29.303000+0000","last_became_active":"2026-03-24T02:13:30.348773+0000","last_became_peered":"2026-03-24T02:13:30.348773+0000","last_unstale":"2026-03-24T02:51:29.303000+0000","last_undegraded":"2026-03-24T02:51:29.303000+0000","last_fullsized":"2026-03-24T02:51:29.303000+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T08:00:33.758016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00018693,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":681,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.300907+0000","last_change":"2026-03-24T02:13:32.406690+0000","last_active":"2026-03-24T02:51:29.300907+0000","last_peered":"2026-03-24T02:51:29.300907+0000","last_clean":"2026-03-24T02:51:29.300907+0000","last_became_active":"2026-03-24T02:13:30.348538+0000","last_became_peered":"2026-03-24T02:13:30.348538+0000","last_unstale":"2026-03-24T02:51:29.300907+0000","last_undegraded":"2026-03-24T02:51:29.300907+0000","last_fullsized":"2026-03-24T02:51:29.300907+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T13:03:07.820972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023985799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":681,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662285+0000","last_change":"2026-03-24T02:13:32.411138+0000","last_active":"2026-03-24T02:51:27.662285+0000","last_peered":"2026-03-24T02:51:27.662285+0000","last_clean":"2026-03-24T02:51:27.662285+0000","last_became_active":"2026-03-24T02:13:30.348335+0000","last_became_peered":"2026-03-24T02:13:30.348335+0000","last_unstale":"2026-03-24T02:51:27.662285+0000","last_undegraded":"2026-03-24T02:51:27.662285+0000","last_fullsized":"2026-03-24T02:51:27.662285+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:37:39.995117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058244000000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'32","reported_seq":726,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662254+0000","last_change":"2026-03-24T02:13:27.310722+0000","last_active":"2026-03-24T02:51:27.662254+0000","last_peered":"2026-03-24T02:51:27.662254+0000","last_clean":"2026-03-24T02:51:27.662254+0000","last_became_active":"2026-03-24T02:13:27.310550+0000","last_became_peered":"2026-03-24T02:13:27.310550+0000","last_unstale":"2026-03-24T02:51:27.662254+0000","last_undegraded":"2026-03-24T02:51:27.662254+0000","last_fullsized":"2026-03-24T02:51:27.662254+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:25.316656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:25.316656+0000","last_clean_scrub_stamp":"2026-03-24T02:13:25.316656+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:46:01.042093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":925696,"data_stored":918560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32,"ondisk_log_size":32,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738833,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1055592,"kb_used_data":2704,"kb_used_omap":1163,"kb_used_meta":1051700,"kb_avail":93316248,"statfs":{"total":96636764160,"available":95555837952,"internally_reserved":0,"allocated":2768896,"data_stored":1461528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1191587,"internal_metadata":1076941149},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738833,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1272888,"kb_used_data":3160,"kb_used_omap":660,"kb_used_meta":1269035,"kb_avail":93098952,"statfs":{"total":96636764160,"available":95333326848,"internally_reserved":0,"allocated":3235840,"data_stored":1920827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":676226,"internal_metadata":1299492478},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738832,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1089456,"kb_used_data":3160,"kb_used_omap":1232,"kb_used_meta":1085039,"kb_avail":93282384,"statfs":{"total":96636764160,"available":95521161216,"internally_reserved":0,"allocated":3235840,"data_stored":1920827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1261597,"internal_metadata":1111080931},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-24T02:51:55.299 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-03-24T02:51:55.453 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:51:55.453 INFO:teuthology.orchestra.run.vm06.stdout:{"epoch":336,"fsid":"91306d06-3c56-4223-8b51-5365acf96311","created":"2026-03-24T02:13:20.206909+0000","modified":"2026-03-24T02:51:27.640956+0000","last_up_change":"2026-03-24T02:13:24.307479+0000","last_in_change":"2026-03-24T02:13:21.167197+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-24T02:13:24.769197+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-03-24T02:13:28.442828+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"7df1aacf-a228-4799-af31-c87d813181b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6809","nonce":178400628},{"type":"v1","addr":"192.168.123.106:6811","nonce":178400628}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6813","nonce":178400628},{"type":"v1","addr":"192.168.123.106:6815","nonce":178400628}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6821","nonce":178400628},{"type":"v1","addr":"192.168.123.106:6823","nonce":178400628}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6817","nonce":178400628},{"type":"v1","addr":"192.168.123.106:6819","nonce":178400628}]},"public_addr":"192.168.123.106:6811/178400628","cluster_addr":"192.168.123.106:6815/178400628","heartbeat_back_addr":"192.168.123.106:6823/178400628","heartbeat_front_addr":"192.168.123.106:6819/178400628","state":["exists","up"]},{"osd":1,"uuid":"ac96531a-b5de-4d84-959b-01dcd619f954","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6800","nonce":2062373619},{"type":"v1","addr":"192.168.123.106:6801","nonce":2062373619}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6802","nonce":2062373619},{"type":"v1","addr":"192.168.123.106:6803","nonce":2062373619}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6806","nonce":2062373619},{"type":"v1","addr":"192.168.123.106:6807","nonce":2062373619}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6804","nonce":2062373619},{"type":"v1","addr":"192.168.123.106:6805","nonce":2062373619}]},"public_addr":"192.168.123.106:6801/2062373619","cluster_addr":"192.168.123.106:6803/2062373619","heartbeat_back_addr":"192.168.123.106:6807/2062373619","heartbeat_front_addr":"192.168.123.106:6805/2062373619","state":["exists","up"]},{"osd":2,"uuid":"57b8ad53-f9a4-4e46-be96-93ad890cbba4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6808","nonce":1501752565},{"type":"v1","addr":"192.168.123.106:6810","nonce":1501752565}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6812","nonce":1501752565},{"type":"v1","addr":"192.168.123.106:6814","nonce":1501752565}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6820","nonce":1501752565},{"type":"v1","addr":"192.168.123.106:6822","nonce":1501752565}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.106:6816","nonce":1501752565},{"type":"v1","addr":"192.168.123.106:6818","nonce":1501752565}]},"public_addr":"192.168.123.106:6810/1501752565","cluster_addr":"192.168.123.106:6814/1501752565","heartbeat_back_addr":"192.168.123.106:6822/1501752565","heartbeat_front_addr":"192.168.123.106:6818/1501752565","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-24T02:13:23.186274+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-24T02:13:23.200078+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-24T02:13:23.214291+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-24T02:51:56.466 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2026-03-24T02:51:56.466 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-03-24T02:51:56.541 INFO:teuthology.orchestra.run.vm06.stdout:{ 2026-03-24T02:51:56.541 INFO:teuthology.orchestra.run.vm06.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-03-24T02:51:56.541 INFO:teuthology.orchestra.run.vm06.stdout:} 2026-03-24T02:51:56.550 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2026-03-24T02:51:56.706 INFO:teuthology.orchestra.run.vm06.stderr:instructed osd(s) 0 to deep-scrub 2026-03-24T02:51:56.718 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2026-03-24T02:51:56.718 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-03-24T02:51:56.794 INFO:teuthology.orchestra.run.vm06.stdout:{ 2026-03-24T02:51:56.794 INFO:teuthology.orchestra.run.vm06.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-03-24T02:51:56.794 INFO:teuthology.orchestra.run.vm06.stdout:} 2026-03-24T02:51:56.804 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2026-03-24T02:51:56.975 INFO:teuthology.orchestra.run.vm06.stderr:instructed osd(s) 1 to deep-scrub 2026-03-24T02:51:56.990 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2026-03-24T02:51:56.990 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-03-24T02:51:57.270 INFO:teuthology.orchestra.run.vm06.stdout:{ 2026-03-24T02:51:57.270 INFO:teuthology.orchestra.run.vm06.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-03-24T02:51:57.270 INFO:teuthology.orchestra.run.vm06.stdout:} 2026-03-24T02:51:57.281 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2026-03-24T02:51:57.438 INFO:teuthology.orchestra.run.vm06.stderr:instructed osd(s) 2 to deep-scrub 2026-03-24T02:51:57.450 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-03-24T02:51:57.601 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:51:57.601 INFO:teuthology.orchestra.run.vm06.stderr:dumped all 2026-03-24T02:51:57.614 INFO:teuthology.orchestra.run.vm06.stdout:{"pg_ready":true,"pg_map":{"version":1489,"stamp":"2026-03-24T02:51:57.205861+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":459299,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":59,"num_write_kb":586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35,"ondisk_log_size":35,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3417936,"kb_used_data":9024,"kb_used_omap":3056,"kb_used_meta":3405775,"kb_avail":279697584,"statfs":{"total":289910292480,"available":286410326016,"internally_reserved":0,"allocated":9240576,"data_stored":5303182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3129410,"internal_metadata":3487514558},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001489"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":679,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.664240+0000","last_change":"2026-03-24T02:13:32.410665+0000","last_active":"2026-03-24T02:51:27.664240+0000","last_peered":"2026-03-24T02:51:27.664240+0000","last_clean":"2026-03-24T02:51:27.664240+0000","last_became_active":"2026-03-24T02:13:30.350133+0000","last_became_peered":"2026-03-24T02:13:30.350133+0000","last_unstale":"2026-03-24T02:51:27.664240+0000","last_undegraded":"2026-03-24T02:51:27.664240+0000","last_fullsized":"2026-03-24T02:51:27.664240+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:51:40.295988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035299000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":679,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662671+0000","last_change":"2026-03-24T02:13:32.410938+0000","last_active":"2026-03-24T02:51:27.662671+0000","last_peered":"2026-03-24T02:51:27.662671+0000","last_clean":"2026-03-24T02:51:27.662671+0000","last_became_active":"2026-03-24T02:13:30.349481+0000","last_became_peered":"2026-03-24T02:13:30.349481+0000","last_unstale":"2026-03-24T02:51:27.662671+0000","last_undegraded":"2026-03-24T02:51:27.662671+0000","last_fullsized":"2026-03-24T02:51:27.662671+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:35:42.157836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055518799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":680,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662534+0000","last_change":"2026-03-24T02:13:32.410738+0000","last_active":"2026-03-24T02:51:27.662534+0000","last_peered":"2026-03-24T02:51:27.662534+0000","last_clean":"2026-03-24T02:51:27.662534+0000","last_became_active":"2026-03-24T02:13:30.348547+0000","last_became_peered":"2026-03-24T02:13:30.348547+0000","last_unstale":"2026-03-24T02:51:27.662534+0000","last_undegraded":"2026-03-24T02:51:27.662534+0000","last_fullsized":"2026-03-24T02:51:27.662534+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:52:23.166796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033254200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":680,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662388+0000","last_change":"2026-03-24T02:13:32.411084+0000","last_active":"2026-03-24T02:51:27.662388+0000","last_peered":"2026-03-24T02:51:27.662388+0000","last_clean":"2026-03-24T02:51:27.662388+0000","last_became_active":"2026-03-24T02:13:30.349564+0000","last_became_peered":"2026-03-24T02:13:30.349564+0000","last_unstale":"2026-03-24T02:51:27.662388+0000","last_undegraded":"2026-03-24T02:51:27.662388+0000","last_fullsized":"2026-03-24T02:51:27.662388+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:04:43.950912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059685599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":681,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.300912+0000","last_change":"2026-03-24T02:13:32.409322+0000","last_active":"2026-03-24T02:51:29.300912+0000","last_peered":"2026-03-24T02:51:29.300912+0000","last_clean":"2026-03-24T02:51:29.300912+0000","last_became_active":"2026-03-24T02:13:30.350751+0000","last_became_peered":"2026-03-24T02:13:30.350751+0000","last_unstale":"2026-03-24T02:51:29.300912+0000","last_undegraded":"2026-03-24T02:51:29.300912+0000","last_fullsized":"2026-03-24T02:51:29.300912+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:58:18.488299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082885000000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":680,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.303000+0000","last_change":"2026-03-24T02:13:32.406625+0000","last_active":"2026-03-24T02:51:29.303000+0000","last_peered":"2026-03-24T02:51:29.303000+0000","last_clean":"2026-03-24T02:51:29.303000+0000","last_became_active":"2026-03-24T02:13:30.348773+0000","last_became_peered":"2026-03-24T02:13:30.348773+0000","last_unstale":"2026-03-24T02:51:29.303000+0000","last_undegraded":"2026-03-24T02:51:29.303000+0000","last_fullsized":"2026-03-24T02:51:29.303000+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T08:00:33.758016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00018693,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":681,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:29.300907+0000","last_change":"2026-03-24T02:13:32.406690+0000","last_active":"2026-03-24T02:51:29.300907+0000","last_peered":"2026-03-24T02:51:29.300907+0000","last_clean":"2026-03-24T02:51:29.300907+0000","last_became_active":"2026-03-24T02:13:30.348538+0000","last_became_peered":"2026-03-24T02:13:30.348538+0000","last_unstale":"2026-03-24T02:51:29.300907+0000","last_undegraded":"2026-03-24T02:51:29.300907+0000","last_fullsized":"2026-03-24T02:51:29.300907+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T13:03:07.820972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023985799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":681,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662285+0000","last_change":"2026-03-24T02:13:32.411138+0000","last_active":"2026-03-24T02:51:27.662285+0000","last_peered":"2026-03-24T02:51:27.662285+0000","last_clean":"2026-03-24T02:51:27.662285+0000","last_became_active":"2026-03-24T02:13:30.348335+0000","last_became_peered":"2026-03-24T02:13:30.348335+0000","last_unstale":"2026-03-24T02:51:27.662285+0000","last_undegraded":"2026-03-24T02:51:27.662285+0000","last_fullsized":"2026-03-24T02:51:27.662285+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:29.337867+0000","last_clean_scrub_stamp":"2026-03-24T02:13:29.337867+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:37:39.995117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058244000000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'32","reported_seq":726,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:27.662254+0000","last_change":"2026-03-24T02:13:27.310722+0000","last_active":"2026-03-24T02:51:27.662254+0000","last_peered":"2026-03-24T02:51:27.662254+0000","last_clean":"2026-03-24T02:51:27.662254+0000","last_became_active":"2026-03-24T02:13:27.310550+0000","last_became_peered":"2026-03-24T02:13:27.310550+0000","last_unstale":"2026-03-24T02:51:27.662254+0000","last_undegraded":"2026-03-24T02:51:27.662254+0000","last_fullsized":"2026-03-24T02:51:27.662254+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:13:25.316656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:13:25.316656+0000","last_clean_scrub_stamp":"2026-03-24T02:13:25.316656+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T11:46:01.042093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":925696,"data_stored":918560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32,"ondisk_log_size":32,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738833,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1055592,"kb_used_data":2704,"kb_used_omap":1163,"kb_used_meta":1051700,"kb_avail":93316248,"statfs":{"total":96636764160,"available":95555837952,"internally_reserved":0,"allocated":2768896,"data_stored":1461528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1191587,"internal_metadata":1076941149},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738834,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1272888,"kb_used_data":3160,"kb_used_omap":660,"kb_used_meta":1269035,"kb_avail":93098952,"statfs":{"total":96636764160,"available":95333326848,"internally_reserved":0,"allocated":3235840,"data_stored":1920827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":676226,"internal_metadata":1299492478},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738832,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1089456,"kb_used_data":3160,"kb_used_omap":1232,"kb_used_meta":1085039,"kb_avail":93282384,"statfs":{"total":96636764160,"available":95521161216,"internally_reserved":0,"allocated":3235840,"data_stored":1920827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1261597,"internal_metadata":1111080931},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-24T02:51:57.614 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-03-24T02:13:29.337867+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=29, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=55, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:57.614 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-03-24T02:13:29.337867+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=29, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=55, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:57.614 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-03-24T02:13:29.337867+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=29, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=55, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:57.614 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-03-24T02:13:29.337867+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=29, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=55, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:57.614 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-03-24T02:13:29.337867+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=29, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=55, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:57.614 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-03-24T02:13:29.337867+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=29, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=55, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:57.614 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-03-24T02:13:29.337867+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=29, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=55, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:57.614 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-03-24T02:13:29.337867+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=29, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=55, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:57.614 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-03-24T02:13:25.316656+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=13, tm_sec=25, tm_wday=1, tm_yday=83, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=24, tm_hour=2, tm_min=51, tm_sec=55, tm_wday=1, tm_yday=83, tm_isdst=0) 2026-03-24T02:51:57.614 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-03-24T02:52:17.615 DEBUG:teuthology.orchestra.run.vm06:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-03-24T02:52:17.771 INFO:teuthology.orchestra.run.vm06.stdout: 2026-03-24T02:52:17.771 INFO:teuthology.orchestra.run.vm06.stderr:dumped all 2026-03-24T02:52:17.783 INFO:teuthology.orchestra.run.vm06.stdout:{"pg_ready":true,"pg_map":{"version":1499,"stamp":"2026-03-24T02:52:17.208355+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":459299,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":59,"num_write_kb":586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3417904,"kb_used_data":9024,"kb_used_omap":3056,"kb_used_meta":3405775,"kb_avail":279697616,"statfs":{"total":289910292480,"available":286410358784,"internally_reserved":0,"allocated":9240576,"data_stored":5303182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3129410,"internal_metadata":3487514558},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001510"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":687,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:52:02.066167+0000","last_change":"2026-03-24T02:52:02.066167+0000","last_active":"2026-03-24T02:52:02.066167+0000","last_peered":"2026-03-24T02:52:02.066167+0000","last_clean":"2026-03-24T02:52:02.066167+0000","last_became_active":"2026-03-24T02:13:30.350133+0000","last_became_peered":"2026-03-24T02:13:30.350133+0000","last_unstale":"2026-03-24T02:52:02.066167+0000","last_undegraded":"2026-03-24T02:52:02.066167+0000","last_fullsized":"2026-03-24T02:52:02.066167+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:52:02.066125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:52:02.066125+0000","last_clean_scrub_stamp":"2026-03-24T02:52:02.066125+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:49:07.023961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035299000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":687,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:52:01.032432+0000","last_change":"2026-03-24T02:52:01.032432+0000","last_active":"2026-03-24T02:52:01.032432+0000","last_peered":"2026-03-24T02:52:01.032432+0000","last_clean":"2026-03-24T02:52:01.032432+0000","last_became_active":"2026-03-24T02:13:30.349481+0000","last_became_peered":"2026-03-24T02:13:30.349481+0000","last_unstale":"2026-03-24T02:52:01.032432+0000","last_undegraded":"2026-03-24T02:52:01.032432+0000","last_fullsized":"2026-03-24T02:52:01.032432+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:52:01.032384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:52:01.032384+0000","last_clean_scrub_stamp":"2026-03-24T02:52:01.032384+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T08:05:36.544378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055518799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":688,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:52:00.056194+0000","last_change":"2026-03-24T02:52:00.056194+0000","last_active":"2026-03-24T02:52:00.056194+0000","last_peered":"2026-03-24T02:52:00.056194+0000","last_clean":"2026-03-24T02:52:00.056194+0000","last_became_active":"2026-03-24T02:13:30.348547+0000","last_became_peered":"2026-03-24T02:13:30.348547+0000","last_unstale":"2026-03-24T02:52:00.056194+0000","last_undegraded":"2026-03-24T02:52:00.056194+0000","last_fullsized":"2026-03-24T02:52:00.056194+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:52:00.056161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:52:00.056161+0000","last_clean_scrub_stamp":"2026-03-24T02:52:00.056161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T14:11:12.781182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033254200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":688,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:59.060757+0000","last_change":"2026-03-24T02:51:59.060757+0000","last_active":"2026-03-24T02:51:59.060757+0000","last_peered":"2026-03-24T02:51:59.060757+0000","last_clean":"2026-03-24T02:51:59.060757+0000","last_became_active":"2026-03-24T02:13:30.349564+0000","last_became_peered":"2026-03-24T02:13:30.349564+0000","last_unstale":"2026-03-24T02:51:59.060757+0000","last_undegraded":"2026-03-24T02:51:59.060757+0000","last_fullsized":"2026-03-24T02:51:59.060757+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:51:59.060704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:51:59.060704+0000","last_clean_scrub_stamp":"2026-03-24T02:51:59.060704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T08:56:47.515194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059685599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"336'3","reported_seq":690,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:57.236163+0000","last_change":"2026-03-24T02:51:57.236163+0000","last_active":"2026-03-24T02:51:57.236163+0000","last_peered":"2026-03-24T02:51:57.236163+0000","last_clean":"2026-03-24T02:51:57.236163+0000","last_became_active":"2026-03-24T02:13:30.350751+0000","last_became_peered":"2026-03-24T02:13:30.350751+0000","last_unstale":"2026-03-24T02:51:57.236163+0000","last_undegraded":"2026-03-24T02:51:57.236163+0000","last_fullsized":"2026-03-24T02:51:57.236163+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"336'3","last_scrub_stamp":"2026-03-24T02:51:57.236105+0000","last_deep_scrub":"336'3","last_deep_scrub_stamp":"2026-03-24T02:51:57.236105+0000","last_clean_scrub_stamp":"2026-03-24T02:51:57.236105+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T07:21:17.679135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082885000000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":688,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:58.634278+0000","last_change":"2026-03-24T02:51:58.634278+0000","last_active":"2026-03-24T02:51:58.634278+0000","last_peered":"2026-03-24T02:51:58.634278+0000","last_clean":"2026-03-24T02:51:58.634278+0000","last_became_active":"2026-03-24T02:13:30.348773+0000","last_became_peered":"2026-03-24T02:13:30.348773+0000","last_unstale":"2026-03-24T02:51:58.634278+0000","last_undegraded":"2026-03-24T02:51:58.634278+0000","last_fullsized":"2026-03-24T02:51:58.634278+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:51:58.634232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:51:58.634232+0000","last_clean_scrub_stamp":"2026-03-24T02:51:58.634232+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:56:35.036226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00018693,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":689,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:57.657156+0000","last_change":"2026-03-24T02:51:57.657156+0000","last_active":"2026-03-24T02:51:57.657156+0000","last_peered":"2026-03-24T02:51:57.657156+0000","last_clean":"2026-03-24T02:51:57.657156+0000","last_became_active":"2026-03-24T02:13:30.348538+0000","last_became_peered":"2026-03-24T02:13:30.348538+0000","last_unstale":"2026-03-24T02:51:57.657156+0000","last_undegraded":"2026-03-24T02:51:57.657156+0000","last_fullsized":"2026-03-24T02:51:57.657156+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-24T02:51:57.657107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-24T02:51:57.657107+0000","last_clean_scrub_stamp":"2026-03-24T02:51:57.657107+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T09:42:49.917891+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0.00023985799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"336'2","reported_seq":690,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:58.052479+0000","last_change":"2026-03-24T02:51:58.052479+0000","last_active":"2026-03-24T02:51:58.052479+0000","last_peered":"2026-03-24T02:51:58.052479+0000","last_clean":"2026-03-24T02:51:58.052479+0000","last_became_active":"2026-03-24T02:13:30.348335+0000","last_became_peered":"2026-03-24T02:13:30.348335+0000","last_unstale":"2026-03-24T02:51:58.052479+0000","last_undegraded":"2026-03-24T02:51:58.052479+0000","last_fullsized":"2026-03-24T02:51:58.052479+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"336'2","last_scrub_stamp":"2026-03-24T02:51:58.052441+0000","last_deep_scrub":"336'2","last_deep_scrub_stamp":"2026-03-24T02:51:58.052441+0000","last_clean_scrub_stamp":"2026-03-24T02:51:58.052441+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:19:30.269254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058244000000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"336'34","reported_seq":736,"reported_epoch":336,"state":"active+clean","last_fresh":"2026-03-24T02:51:57.033687+0000","last_change":"2026-03-24T02:51:57.033687+0000","last_active":"2026-03-24T02:51:57.033687+0000","last_peered":"2026-03-24T02:51:57.033687+0000","last_clean":"2026-03-24T02:51:57.033687+0000","last_became_active":"2026-03-24T02:13:27.310550+0000","last_became_peered":"2026-03-24T02:13:27.310550+0000","last_unstale":"2026-03-24T02:51:57.033687+0000","last_undegraded":"2026-03-24T02:51:57.033687+0000","last_fullsized":"2026-03-24T02:51:57.033687+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"336'34","last_scrub_stamp":"2026-03-24T02:51:57.033606+0000","last_deep_scrub":"336'34","last_deep_scrub_stamp":"2026-03-24T02:51:57.033606+0000","last_clean_scrub_stamp":"2026-03-24T02:51:57.033606+0000","objects_scrubbed":2,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:39:13.000105+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":459280,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":46,"num_read_kb":37,"num_write":57,"num_write_kb":584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":925696,"data_stored":918560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":34,"ondisk_log_size":34,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738837,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1055584,"kb_used_data":2704,"kb_used_omap":1163,"kb_used_meta":1051700,"kb_avail":93316256,"statfs":{"total":96636764160,"available":95555846144,"internally_reserved":0,"allocated":2768896,"data_stored":1461528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1191587,"internal_metadata":1076941149},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738838,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1272872,"kb_used_data":3160,"kb_used_omap":660,"kb_used_meta":1269035,"kb_avail":93098968,"statfs":{"total":96636764160,"available":95333343232,"internally_reserved":0,"allocated":3235840,"data_stored":1920827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":676226,"internal_metadata":1299492478},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738836,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1089448,"kb_used_data":3160,"kb_used_omap":1232,"kb_used_meta":1085039,"kb_avail":93282392,"statfs":{"total":96636764160,"available":95521169408,"internally_reserved":0,"allocated":3235840,"data_stored":1920827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1261597,"internal_metadata":1111080931},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":462848,"data_stored":459280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-24T02:52:17.784 DEBUG:teuthology.orchestra.run.vm06:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2026-03-24T02:52:17.964 INFO:teuthology.misc:Shutting down mds daemons... 2026-03-24T02:52:17.964 INFO:teuthology.misc:Shutting down osd daemons... 2026-03-24T02:52:17.964 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2026-03-24T02:52:17.964 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:52:18.042 INFO:tasks.ceph.cluster1.osd.0:Stopped 2026-03-24T02:52:18.042 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2026-03-24T02:52:18.043 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:52:21.155 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.1 is failed for ~0s 2026-03-24T02:52:24.144 INFO:tasks.ceph.cluster1.osd.1:Stopped 2026-03-24T02:52:24.144 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2026-03-24T02:52:24.144 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:52:24.231 INFO:tasks.ceph.cluster1.osd.2:Stopped 2026-03-24T02:52:24.231 INFO:teuthology.misc:Shutting down mgr daemons... 2026-03-24T02:52:24.231 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2026-03-24T02:52:24.231 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:52:24.259 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2026-03-24T02:52:24.259 INFO:teuthology.misc:Shutting down mon daemons... 2026-03-24T02:52:24.259 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2026-03-24T02:52:24.259 INFO:teuthology.orchestra.run:waiting for 300 2026-03-24T02:52:24.310 INFO:tasks.ceph.cluster1.mon.a:Stopped 2026-03-24T02:52:24.310 INFO:tasks.ceph:Checking cluster log for badness... 2026-03-24T02:52:24.310 DEBUG:teuthology.orchestra.run.vm06:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-03-24T02:52:24.360 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@vm06.local 2026-03-24T02:52:24.360 DEBUG:teuthology.orchestra.run.vm06:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2026-03-24T02:52:24.614 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@vm06.local 2026-03-24T02:52:24.614 DEBUG:teuthology.orchestra.run.vm06:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2026-03-24T02:52:24.807 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@vm06.local 2026-03-24T02:52:24.807 DEBUG:teuthology.orchestra.run.vm06:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2026-03-24T02:52:24.986 INFO:tasks.ceph:Archiving mon data... 2026-03-24T02:52:24.987 DEBUG:teuthology.misc:Transferring archived files from vm06:/var/lib/ceph/mon/cluster1-a to /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3539/data/cluster1.mon.a.tgz 2026-03-24T02:52:24.987 DEBUG:teuthology.orchestra.run.vm06:> mktemp 2026-03-24T02:52:24.990 INFO:teuthology.orchestra.run.vm06.stdout:/tmp/tmp.XmsQVVl2N4 2026-03-24T02:52:24.990 DEBUG:teuthology.orchestra.run.vm06:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.XmsQVVl2N4 2026-03-24T02:52:25.092 DEBUG:teuthology.orchestra.run.vm06:> sudo chmod 0666 /tmp/tmp.XmsQVVl2N4 2026-03-24T02:52:25.147 DEBUG:teuthology.orchestra.remote:vm06:/tmp/tmp.XmsQVVl2N4 is 401KB 2026-03-24T02:52:25.195 DEBUG:teuthology.orchestra.run.vm06:> rm -fr /tmp/tmp.XmsQVVl2N4 2026-03-24T02:52:25.197 INFO:tasks.ceph:Cleaning ceph cluster... 2026-03-24T02:52:25.197 DEBUG:teuthology.orchestra.run.vm06:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-03-24T02:52:25.239 DEBUG:teuthology.orchestra.run.vm07:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-03-24T02:52:25.290 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-03-24T02:52:25.294 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-03-24T02:52:25.294 INFO:tasks.ceph:Archiving crash dumps... 2026-03-24T02:52:25.295 DEBUG:teuthology.misc:Transferring archived files from vm06:/var/lib/ceph/crash to /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3539/remote/vm06/crash 2026-03-24T02:52:25.295 DEBUG:teuthology.orchestra.run.vm06:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-03-24T02:52:25.339 DEBUG:teuthology.misc:Transferring archived files from vm07:/var/lib/ceph/crash to /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3539/remote/vm07/crash 2026-03-24T02:52:25.339 DEBUG:teuthology.orchestra.run.vm07:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-03-24T02:52:25.345 INFO:tasks.ceph:Compressing logs... 2026-03-24T02:52:25.346 DEBUG:teuthology.orchestra.run.vm06:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-03-24T02:52:25.383 DEBUG:teuthology.orchestra.run.vm07:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-03-24T02:52:25.390 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26874.log 2026-03-24T02:52:25.391 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27134.log 2026-03-24T02:52:25.391 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.26874.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26874.log.gz 2026-03-24T02:52:25.391 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22926.log 2026-03-24T02:52:25.391 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.27134.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.27134.log.gz 2026-03-24T02:52:25.391 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2026-03-24T02:52:25.392 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.22926.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.22926.log.gz 2026-03-24T02:52:25.392 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26120.log 2026-03-24T02:52:25.392 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26952.log 2026-03-24T02:52:25.392 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.26120.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26120.log.gz 2026-03-24T02:52:25.395 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25945.log 2026-03-24T02:52:25.395 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.26952.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26952.log.gz 2026-03-24T02:52:25.395 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22968.log 2026-03-24T02:52:25.395 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.25945.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25945.log.gz 2026-03-24T02:52:25.396 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.log 2026-03-24T02:52:25.396 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.22968.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.22968.log.gz 2026-03-24T02:52:25.396 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22854.log 2026-03-24T02:52:25.396 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27004.log 2026-03-24T02:52:25.396 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.22854.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.22854.log.gz 2026-03-24T02:52:25.397 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27030.log 2026-03-24T02:52:25.397 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2026-03-24T02:52:25.397 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.27004.log: /var/log/ceph/cluster1-client.admin.27030.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.27030.log.gz 2026-03-24T02:52:25.397 INFO:teuthology.orchestra.run.vm06.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.27004.log.gz 2026-03-24T02:52:25.397 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25867.log 2026-03-24T02:52:25.398 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22992.log 2026-03-24T02:52:25.398 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-osd.0.log: /var/log/ceph/cluster1-client.admin.25867.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25867.log.gz 2026-03-24T02:52:25.398 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27184.log 2026-03-24T02:52:25.398 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25704.log 2026-03-24T02:52:25.399 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.27184.log: 87.4% -- replaced with /var/log/ceph/cluster1.log.gz 2026-03-24T02:52:25.399 INFO:teuthology.orchestra.run.vm06.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.27184.log.gz 2026-03-24T02:52:25.399 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45143.log 2026-03-24T02:52:25.399 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.22992.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.22992.log.gz 2026-03-24T02:52:25.399 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89473.log 2026-03-24T02:52:25.399 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45143.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56871.log 2026-03-24T02:52:25.400 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25946.log 2026-03-24T02:52:25.400 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.25704.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26900.log 2026-03-24T02:52:25.400 INFO:teuthology.orchestra.run.vm06.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25704.log.gz 2026-03-24T02:52:25.400 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.25946.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25946.log.gz 2026-03-24T02:52:25.400 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25730.log 2026-03-24T02:52:25.400 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.89473.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.89473.log.gz 2026-03-24T02:52:25.400 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26169.log 2026-03-24T02:52:25.400 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37151.log 2026-03-24T02:52:25.401 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56871.log: /var/log/ceph/cluster1-client.admin.26169.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68280.log 2026-03-24T02:52:25.401 INFO:teuthology.orchestra.run.vm07.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26169.log.gz 2026-03-24T02:52:25.401 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.26900.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25756.log 2026-03-24T02:52:25.401 INFO:teuthology.orchestra.run.vm06.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26900.log.gz 2026-03-24T02:52:25.401 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.25730.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25730.log.gz 2026-03-24T02:52:25.401 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27056.log 2026-03-24T02:52:25.401 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.25756.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23017.log 2026-03-24T02:52:25.401 INFO:teuthology.orchestra.run.vm06.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25756.log.gz 2026-03-24T02:52:25.401 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.27056.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.27056.log.gz 2026-03-24T02:52:25.402 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26146.log 2026-03-24T02:52:25.402 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.23017.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25789.log 2026-03-24T02:52:25.402 INFO:teuthology.orchestra.run.vm06.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.23017.log.gz 2026-03-24T02:52:25.402 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.26146.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26146.log.gz 2026-03-24T02:52:25.402 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37151.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56871.log.gz 2026-03-24T02:52:25.402 INFO:teuthology.orchestra.run.vm07.stderr: 91.2% -- replaced with /var/log/ceph/cluster2-client.mirror.45143.log.gz 2026-03-24T02:52:25.402 INFO:teuthology.orchestra.run.vm07.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.37151.log.gz 2026-03-24T02:52:25.402 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47699.log 2026-03-24T02:52:25.402 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68280.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43714.log 2026-03-24T02:52:25.402 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47699.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44541.log 2026-03-24T02:52:25.402 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.68280.log.gz 2026-03-24T02:52:25.402 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.43714.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47699.log.gz 2026-03-24T02:52:25.403 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36686.log 2026-03-24T02:52:25.403 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.44541.log: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43714.log.gz 2026-03-24T02:52:25.403 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57372.log 2026-03-24T02:52:25.403 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25788.log 2026-03-24T02:52:25.403 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.25789.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.19232.log 2026-03-24T02:52:25.403 INFO:teuthology.orchestra.run.vm06.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25789.log.gz 2026-03-24T02:52:25.403 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.25788.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25788.log.gz 2026-03-24T02:52:25.403 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25790.log 2026-03-24T02:52:25.403 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/ceph.tmp-client.admin.19232.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2026-03-24T02:52:25.403 INFO:teuthology.orchestra.run.vm06.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.19232.log.gz 2026-03-24T02:52:25.403 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.25790.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25790.log.gz 2026-03-24T02:52:25.403 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27082.log 2026-03-24T02:52:25.403 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25658.log 2026-03-24T02:52:25.403 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.27082.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.27082.log.gz 2026-03-24T02:52:25.404 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2026-03-24T02:52:25.404 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44733.log 2026-03-24T02:52:25.404 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36686.log: /var/log/ceph/cluster1-client.mirror.57372.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27324.log 2026-03-24T02:52:25.404 INFO:teuthology.orchestra.run.vm07.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.36686.log.gz 2026-03-24T02:52:25.404 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.26534.log 2026-03-24T02:52:25.404 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25608.log 2026-03-24T02:52:25.404 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.27324.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80016.log 2026-03-24T02:52:25.404 INFO:teuthology.orchestra.run.vm07.stderr: 64.1% -- replaced with /var/log/ceph/cluster2-client.mirror.27324.log.gz 2026-03-24T02:52:25.404 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.2.26534.log: 92.4%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.57372.log.gz -5 2026-03-24T02:52:25.404 INFO:teuthology.orchestra.run.vm07.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.46856.log 2026-03-24T02:52:25.405 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80016.log: /var/log/ceph/cluster2-client.mirror.44733.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.80016.log.gz 2026-03-24T02:52:25.405 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26042.log 2026-03-24T02:52:25.405 INFO:teuthology.orchestra.run.vm07.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44733.log.gz 2026-03-24T02:52:25.406 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46856.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59467.log 2026-03-24T02:52:25.406 INFO:teuthology.orchestra.run.vm07.stderr: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46856.log.gz 2026-03-24T02:52:25.406 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.26042.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.26042.log.gz 2026-03-24T02:52:25.406 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86751.log 2026-03-24T02:52:25.406 INFO:teuthology.orchestra.run.vm07.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.44541.log.gz 2026-03-24T02:52:25.406 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.59467.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.27307.log 2026-03-24T02:52:25.406 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86751.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71522.log 2026-03-24T02:52:25.407 INFO:teuthology.orchestra.run.vm07.stderr: 83.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86751.log.gz 2026-03-24T02:52:25.407 INFO:teuthology.orchestra.run.vm07.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.59467.log.gz 2026-03-24T02:52:25.407 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.27307.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38334.log 2026-03-24T02:52:25.407 INFO:teuthology.orchestra.run.vm07.stderr: 72.6% -- replaced with /var/log/ceph/cluster2-client.admin.27307.log.gz 2026-03-24T02:52:25.407 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82702.log 2026-03-24T02:52:25.407 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71522.log: /var/log/ceph/cluster2-client.mirror.38334.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36779.log 2026-03-24T02:52:25.407 INFO:teuthology.orchestra.run.vm07.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.2.26534.log.gz 2026-03-24T02:52:25.407 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71522.log.gz 2026-03-24T02:52:25.408 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82702.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38334.log.gz 2026-03-24T02:52:25.408 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82702.log.gz 2026-03-24T02:52:25.408 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.89258.log 2026-03-24T02:52:25.408 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36779.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.29193.log 2026-03-24T02:52:25.408 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.89258.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38969.log 2026-03-24T02:52:25.408 INFO:teuthology.orchestra.run.vm07.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.89258.log.gz 2026-03-24T02:52:25.408 INFO:teuthology.orchestra.run.vm07.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.36779.log.gz 2026-03-24T02:52:25.408 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.1.29193.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67667.log 2026-03-24T02:52:25.409 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79002.log 2026-03-24T02:52:25.409 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38969.log: /var/log/ceph/cluster2-client.mirror.67667.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.38969.log.gz 2026-03-24T02:52:25.409 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.67667.log.gz 2026-03-24T02:52:25.410 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53083.log 2026-03-24T02:52:25.410 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79002.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43036.log 2026-03-24T02:52:25.410 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79002.log.gz 2026-03-24T02:52:25.410 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53083.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87712.log 2026-03-24T02:52:25.411 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79277.log 2026-03-24T02:52:25.411 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43036.log: 88.3%/var/log/ceph/cluster2-client.mirror.87712.log: -- replaced with /var/log/ceph/cluster2-client.mirror.53083.log.gz 2026-03-24T02:52:25.411 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53636.log 2026-03-24T02:52:25.411 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.87712.log.gz 2026-03-24T02:52:25.411 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27392.log 2026-03-24T02:52:25.411 INFO:teuthology.orchestra.run.vm07.stderr: 94.2% -- replaced with /var/log/ceph/cluster1-client.mirror.1.29193.log.gz 2026-03-24T02:52:25.411 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.53636.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25676.log 2026-03-24T02:52:25.412 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.27392.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32043.log 2026-03-24T02:52:25.412 INFO:teuthology.orchestra.run.vm07.stderr: 68.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27392.log.gz 2026-03-24T02:52:25.412 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.25676.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.53636.log.gz 2026-03-24T02:52:25.412 INFO:teuthology.orchestra.run.vm07.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25676.log.gz 2026-03-24T02:52:25.412 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89330.log 2026-03-24T02:52:25.412 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32043.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55765.log 2026-03-24T02:52:25.413 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.89330.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33075.log 2026-03-24T02:52:25.413 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32043.log.gz 2026-03-24T02:52:25.413 INFO:teuthology.orchestra.run.vm07.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.89330.log.gz 2026-03-24T02:52:25.413 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55765.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40486.log 2026-03-24T02:52:25.413 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33075.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29051.log 2026-03-24T02:52:25.413 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40486.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33075.log.gz 2026-03-24T02:52:25.414 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87841.log 2026-03-24T02:52:25.414 INFO:teuthology.orchestra.run.vm07.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55765.log.gz 2026-03-24T02:52:25.414 INFO:teuthology.orchestra.run.vm07.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40486.log.gz 2026-03-24T02:52:25.414 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29051.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43036.log.gz 2026-03-24T02:52:25.414 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79277.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81062.log 2026-03-24T02:52:25.414 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80784.log 2026-03-24T02:52:25.415 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81062.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66045.log 2026-03-24T02:52:25.415 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25633.log 2026-03-24T02:52:25.415 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80784.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79277.log.gz 2026-03-24T02:52:25.415 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.25608.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25608.log.gz 2026-03-24T02:52:25.415 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25938.log 2026-03-24T02:52:25.416 INFO:teuthology.orchestra.run.vm07.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81062.log.gz 2026-03-24T02:52:25.416 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.25633.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25633.log.gz 2026-03-24T02:52:25.416 INFO:teuthology.orchestra.run.vm07.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80784.log.gz 2026-03-24T02:52:25.416 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44127.log 2026-03-24T02:52:25.416 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88625.log 2026-03-24T02:52:25.417 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66045.log: /var/log/ceph/cluster2-client.mirror.44127.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26369.log 2026-03-24T02:52:25.417 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66045.log.gz/var/log/ceph/cluster1-client.mirror.88625.log: 2026-03-24T02:52:25.417 INFO:teuthology.orchestra.run.vm07.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88625.log.gz 2026-03-24T02:52:25.417 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40242.log 2026-03-24T02:52:25.417 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74093.log 2026-03-24T02:52:25.417 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.26369.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.26369.log.gz 2026-03-24T02:52:25.418 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40242.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60062.log 2026-03-24T02:52:25.418 INFO:teuthology.orchestra.run.vm07.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40242.log.gz 2026-03-24T02:52:25.418 INFO:teuthology.orchestra.run.vm07.stderr: 96.8% -- replaced with /var/log/ceph/cluster2-client.mirror.29051.log.gz 2026-03-24T02:52:25.418 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.74093.log: /var/log/ceph/cluster2-client.mirror.87841.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.74093.log.gz 2026-03-24T02:52:25.418 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40951.log 2026-03-24T02:52:25.419 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81419.log 2026-03-24T02:52:25.419 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.60062.log: /var/log/ceph/cluster2-client.mirror.40951.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.60062.log.gz 2026-03-24T02:52:25.419 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26826.log 2026-03-24T02:52:25.419 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.25658.log: 80.4% -- replaced with /var/log/ceph/cluster1-client.admin.25658.log.gz 2026-03-24T02:52:25.419 INFO:teuthology.orchestra.run.vm07.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40951.log.gz 2026-03-24T02:52:25.419 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26978.log 2026-03-24T02:52:25.420 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81419.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.81419.log.gz 2026-03-24T02:52:25.420 INFO:teuthology.orchestra.run.vm07.stderr: 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.44127.log.gz 2026-03-24T02:52:25.420 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33168.log 2026-03-24T02:52:25.420 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.26826.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73039.log 2026-03-24T02:52:25.420 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55032.log 2026-03-24T02:52:25.420 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33168.log: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.26826.log.gz 2026-03-24T02:52:25.421 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.73039.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65058.log 2026-03-24T02:52:25.421 INFO:teuthology.orchestra.run.vm07.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33168.log.gz 2026-03-24T02:52:25.421 INFO:teuthology.orchestra.run.vm07.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.73039.log.gz 2026-03-24T02:52:25.421 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.55032.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36407.log 2026-03-24T02:52:25.421 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73131.log 2026-03-24T02:52:25.421 INFO:teuthology.orchestra.run.vm07.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.55032.log.gz 2026-03-24T02:52:25.422 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36407.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38729.log 2026-03-24T02:52:25.422 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65058.log: /var/log/ceph/cluster2-client.mirror.73131.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.36407.log.gz 2026-03-24T02:52:25.422 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.65058.log.gz 2026-03-24T02:52:25.422 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41963.log 2026-03-24T02:52:25.422 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38729.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29074.log 2026-03-24T02:52:25.423 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41963.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37372.log 2026-03-24T02:52:25.423 INFO:teuthology.orchestra.run.vm07.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.73131.log.gz 2026-03-24T02:52:25.423 INFO:teuthology.orchestra.run.vm07.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.41963.log.gz 2026-03-24T02:52:25.426 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29074.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46251.log 2026-03-24T02:52:25.427 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37372.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.37372.log.gz 2026-03-24T02:52:25.427 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.25938.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25938.log.gz 2026-03-24T02:52:25.427 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27160.log 2026-03-24T02:52:25.428 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.26978.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26978.log.gz 2026-03-24T02:52:25.430 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34275.log 2026-03-24T02:52:25.431 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46251.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46251.log.gz 2026-03-24T02:52:25.434 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58992.log 2026-03-24T02:52:25.435 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26024.log 2026-03-24T02:52:25.435 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.27160.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.27160.log.gz 2026-03-24T02:52:25.436 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23052.log 2026-03-24T02:52:25.436 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.26024.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26024.log.gz 2026-03-24T02:52:25.437 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34275.log: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.34275.log.gz 2026-03-24T02:52:25.438 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63912.log 2026-03-24T02:52:25.439 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.58992.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58992.log.gz 2026-03-24T02:52:25.442 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68697.log 2026-03-24T02:52:25.443 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.63912.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63912.log.gz 2026-03-24T02:52:25.446 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60096.log 2026-03-24T02:52:25.447 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2026-03-24T02:52:25.447 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.23052.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.23052.log.gz 2026-03-24T02:52:25.449 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68697.log: 96.5% -- replaced with /var/log/ceph/cluster2-client.mirror.68697.log.gz 2026-03-24T02:52:25.450 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80419.log 2026-03-24T02:52:25.451 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26016.log 2026-03-24T02:52:25.451 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.60096.log: 94.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60096.log.gz 2026-03-24T02:52:25.454 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64723.log 2026-03-24T02:52:25.455 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80419.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80419.log.gz 2026-03-24T02:52:25.458 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42489.log 2026-03-24T02:52:25.458 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64723.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.64723.log.gz 2026-03-24T02:52:25.462 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81468.log 2026-03-24T02:52:25.463 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25868.log 2026-03-24T02:52:25.463 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.26016.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26016.log.gz 2026-03-24T02:52:25.464 INFO:teuthology.orchestra.run.vm06.stderr: 93.1% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2026-03-24T02:52:25.475 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23181.log 2026-03-24T02:52:25.475 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.25868.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25868.log.gz 2026-03-24T02:52:25.475 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26094.log 2026-03-24T02:52:25.475 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.23181.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.23181.log.gz 2026-03-24T02:52:25.478 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42489.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80216.log 2026-03-24T02:52:25.479 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81468.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.81468.log.gz 2026-03-24T02:52:25.490 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68183.log 2026-03-24T02:52:25.491 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25860.log 2026-03-24T02:52:25.491 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.26094.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26094.log.gz 2026-03-24T02:52:25.491 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80216.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80216.log.gz 2026-03-24T02:52:25.491 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81561.log 2026-03-24T02:52:25.498 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68183.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.68183.log.gz 2026-03-24T02:52:25.498 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70734.log 2026-03-24T02:52:25.503 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26023.log 2026-03-24T02:52:25.503 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.25860.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25860.log.gz 2026-03-24T02:52:25.506 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81561.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.81561.log.gz 2026-03-24T02:52:25.506 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53661.log 2026-03-24T02:52:25.511 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2026-03-24T02:52:25.511 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.26023.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26023.log.gz 2026-03-24T02:52:25.514 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.70734.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70734.log.gz 2026-03-24T02:52:25.515 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61608.log 2026-03-24T02:52:25.516 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53661.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53661.log.gz 2026-03-24T02:52:25.519 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27108.log 2026-03-24T02:52:25.526 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83010.log 2026-03-24T02:52:25.527 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61608.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61608.log.gz 2026-03-24T02:52:25.527 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89551.log 2026-03-24T02:52:25.535 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2026-03-24T02:52:25.535 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.83010.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.83010.log.gz 2026-03-24T02:52:25.535 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.27108.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.27108.log.gz 2026-03-24T02:52:25.535 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84961.log 2026-03-24T02:52:25.535 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.89551.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.89551.log.gz 2026-03-24T02:52:25.543 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26926.log 2026-03-24T02:52:25.546 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44670.log 2026-03-24T02:52:25.546 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84961.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84961.log.gz 2026-03-24T02:52:25.549 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38729.log.gz 2026-03-24T02:52:25.549 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27358.log 2026-03-24T02:52:25.550 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44670.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44670.log.gz 2026-03-24T02:52:25.550 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31357.log 2026-03-24T02:52:25.550 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.27358.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.27358.log.gz 2026-03-24T02:52:25.551 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48086.log 2026-03-24T02:52:25.551 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25678.log 2026-03-24T02:52:25.551 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.26926.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26926.log.gz 2026-03-24T02:52:25.551 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31357.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31357.log.gz 2026-03-24T02:52:25.551 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26194.log 2026-03-24T02:52:25.552 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.48086.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53160.log 2026-03-24T02:52:25.562 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.26194.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26194.log.gz 2026-03-24T02:52:25.563 INFO:teuthology.orchestra.run.vm06.stderr:/var/log/ceph/cluster1-client.admin.25678.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25678.log.gz 2026-03-24T02:52:25.566 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67621.log 2026-03-24T02:52:25.570 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53160.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53160.log.gz 2026-03-24T02:52:25.578 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75622.log 2026-03-24T02:52:25.579 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.67621.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.67621.log.gz 2026-03-24T02:52:25.579 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26294.log 2026-03-24T02:52:25.586 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.75622.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75622.log.gz 2026-03-24T02:52:25.587 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33912.log 2026-03-24T02:52:25.587 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.26294.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.26294.log.gz 2026-03-24T02:52:25.595 INFO:teuthology.orchestra.run.vm07.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.87841.log.gz 2026-03-24T02:52:25.595 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31543.log 2026-03-24T02:52:25.596 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33912.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.33912.log.gz 2026-03-24T02:52:25.596 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77742.log 2026-03-24T02:52:25.596 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31543.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31543.log.gz 2026-03-24T02:52:25.596 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34646.log 2026-03-24T02:52:25.597 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77742.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77742.log.gz 2026-03-24T02:52:25.597 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61810.log 2026-03-24T02:52:25.597 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34646.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74228.log 2026-03-24T02:52:25.597 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.34646.log.gz 2026-03-24T02:52:25.598 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61810.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.61810.log.gz 2026-03-24T02:52:25.598 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71030.log 2026-03-24T02:52:25.598 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.74228.log: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42489.log.gz 2026-03-24T02:52:25.599 INFO:teuthology.orchestra.run.vm07.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74228.log.gz 2026-03-24T02:52:25.599 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25858.log 2026-03-24T02:52:25.600 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71030.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45485.log 2026-03-24T02:52:25.600 INFO:teuthology.orchestra.run.vm07.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.71030.log.gz 2026-03-24T02:52:25.600 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.25858.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25858.log.gz 2026-03-24T02:52:25.600 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86730.log 2026-03-24T02:52:25.600 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45485.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69406.log 2026-03-24T02:52:25.601 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86730.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45485.log.gz 2026-03-24T02:52:25.601 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38523.log 2026-03-24T02:52:25.601 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.69406.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69406.log.gz 2026-03-24T02:52:25.602 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57327.log 2026-03-24T02:52:25.602 INFO:teuthology.orchestra.run.vm07.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86730.log.gz 2026-03-24T02:52:25.602 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38523.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49189.log 2026-03-24T02:52:25.602 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.57327.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38523.log.gz 2026-03-24T02:52:25.602 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.57327.log.gz 2026-03-24T02:52:25.602 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44649.log 2026-03-24T02:52:25.603 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.49189.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61330.log 2026-03-24T02:52:25.603 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84646.log 2026-03-24T02:52:25.604 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61330.log: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61330.log.gz 2026-03-24T02:52:25.604 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78193.log 2026-03-24T02:52:25.605 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84646.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84646.log.gz 2026-03-24T02:52:25.605 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35419.log 2026-03-24T02:52:25.605 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78193.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.78193.log.gz 2026-03-24T02:52:25.606 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.26528.log 2026-03-24T02:52:25.606 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35419.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.35419.log.gz 2026-03-24T02:52:25.606 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79416.log 2026-03-24T02:52:25.607 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.3.26528.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59492.log 2026-03-24T02:52:25.615 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44649.log: /var/log/ceph/cluster2-client.mirror.79416.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44649.log.gz 2026-03-24T02:52:25.621 INFO:teuthology.orchestra.run.vm07.stderr: 96.0% -- replaced with /var/log/ceph/cluster2-client.mirror.29074.log.gz 2026-03-24T02:52:25.622 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79416.log.gz 2026-03-24T02:52:25.622 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28609.log 2026-03-24T02:52:25.622 INFO:teuthology.orchestra.run.vm07.stderr: 96.0% -- replaced with /var/log/ceph/cluster2-client.mirror.48086.log.gz 2026-03-24T02:52:25.623 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59492.log: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59492.log.gz 2026-03-24T02:52:25.624 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33040.log 2026-03-24T02:52:25.624 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72647.log 2026-03-24T02:52:25.624 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.28609.log: /var/log/ceph/cluster2-client.mirror.33040.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.28609.log.gz 2026-03-24T02:52:25.624 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65306.log 2026-03-24T02:52:25.624 INFO:teuthology.orchestra.run.vm07.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33040.log.gz 2026-03-24T02:52:25.625 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72647.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72647.log.gz 2026-03-24T02:52:25.625 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25656.log 2026-03-24T02:52:25.625 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28468.log 2026-03-24T02:52:25.626 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65306.log: /var/log/ceph/cluster2-client.admin.25656.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.65306.log.gz 2026-03-24T02:52:25.626 INFO:teuthology.orchestra.run.vm07.stderr: 80.3% -- replaced with /var/log/ceph/cluster2-client.admin.25656.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48427.log 2026-03-24T02:52:25.626 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-24T02:52:25.626 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.28468.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80971.log 2026-03-24T02:52:25.627 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54240.log 2026-03-24T02:52:25.627 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80971.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80971.log.gz 2026-03-24T02:52:25.630 INFO:teuthology.orchestra.run.vm06.stderr: 91.4% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2026-03-24T02:52:25.638 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.48427.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61992.log 2026-03-24T02:52:25.647 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54240.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75217.log 2026-03-24T02:52:25.647 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70556.log 2026-03-24T02:52:25.647 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.61992.log: /var/log/ceph/cluster1-client.mirror.75217.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34232.log 2026-03-24T02:52:25.647 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.70556.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83408.log 2026-03-24T02:52:25.648 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34232.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43416.log 2026-03-24T02:52:25.648 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83408.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26734.log 2026-03-24T02:52:25.648 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43416.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66765.log 2026-03-24T02:52:25.648 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.26734.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88710.log 2026-03-24T02:52:25.648 INFO:teuthology.orchestra.run.vm07.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26734.log.gz 2026-03-24T02:52:25.649 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.66765.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81539.log 2026-03-24T02:52:25.649 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88710.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41633.log 2026-03-24T02:52:25.649 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81539.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39412.log 2026-03-24T02:52:25.649 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41633.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89395.log 2026-03-24T02:52:25.650 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39412.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48513.log 2026-03-24T02:52:25.650 INFO:teuthology.orchestra.run.vm07.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88710.log.gz 2026-03-24T02:52:25.650 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53237.log 2026-03-24T02:52:25.650 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.89395.log: /var/log/ceph/cluster2-client.mirror.48513.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89603.log 2026-03-24T02:52:25.655 INFO:teuthology.orchestra.run.vm07.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.89395.log.gz 2026-03-24T02:52:25.655 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70556.log.gz 2026-03-24T02:52:25.655 INFO:teuthology.orchestra.run.vm07.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.81539.log.gz 2026-03-24T02:52:25.655 INFO:teuthology.orchestra.run.vm07.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.43416.log.gz 2026-03-24T02:52:25.655 INFO:teuthology.orchestra.run.vm07.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.66765.log.gz 2026-03-24T02:52:25.658 INFO:teuthology.orchestra.run.vm07.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.83408.log.gz 2026-03-24T02:52:25.659 INFO:teuthology.orchestra.run.vm07.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.61992.log.gz 2026-03-24T02:52:25.659 INFO:teuthology.orchestra.run.vm07.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.75217.log.gz 2026-03-24T02:52:25.659 INFO:teuthology.orchestra.run.vm07.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54240.log.gz 2026-03-24T02:52:25.660 INFO:teuthology.orchestra.run.vm07.stderr: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.34232.log.gz 2026-03-24T02:52:25.661 INFO:teuthology.orchestra.run.vm07.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39412.log.gz 2026-03-24T02:52:25.669 INFO:teuthology.orchestra.run.vm07.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.28468.log.gz 2026-03-24T02:52:25.670 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.53237.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83543.log 2026-03-24T02:52:25.670 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.89603.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.89603.log.gz 2026-03-24T02:52:25.671 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25936.log 2026-03-24T02:52:25.671 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83543.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83543.log.gz 2026-03-24T02:52:25.671 INFO:teuthology.orchestra.run.vm07.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.53237.log.gz 2026-03-24T02:52:25.671 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68436.log 2026-03-24T02:52:25.672 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.25936.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25936.log.gz 2026-03-24T02:52:25.672 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89212.log 2026-03-24T02:52:25.672 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68436.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.68436.log.gz 2026-03-24T02:52:25.673 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81944.log 2026-03-24T02:52:25.673 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.89212.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.89212.log.gz 2026-03-24T02:52:25.673 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76477.log 2026-03-24T02:52:25.674 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81944.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81944.log.gz 2026-03-24T02:52:25.674 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64828.log 2026-03-24T02:52:25.676 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76477.log: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76477.log.gz 2026-03-24T02:52:25.676 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73387.log 2026-03-24T02:52:25.676 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64828.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64828.log.gz 2026-03-24T02:52:25.676 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78618.log 2026-03-24T02:52:25.677 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.73387.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.73387.log.gz 2026-03-24T02:52:25.677 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82651.log 2026-03-24T02:52:25.678 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78618.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.78618.log.gz 2026-03-24T02:52:25.678 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45227.log 2026-03-24T02:52:25.679 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82651.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82651.log.gz 2026-03-24T02:52:25.679 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.27582.log 2026-03-24T02:52:25.680 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.45227.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.45227.log.gz 2026-03-24T02:52:25.680 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57868.log 2026-03-24T02:52:25.681 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.0.27582.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65710.log 2026-03-24T02:52:25.684 INFO:teuthology.orchestra.run.vm07.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.3.26528.log.gz 2026-03-24T02:52:25.686 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57868.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57868.log.gz 2026-03-24T02:52:25.687 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37927.log 2026-03-24T02:52:25.698 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65710.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65710.log.gz 2026-03-24T02:52:25.698 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88178.log 2026-03-24T02:52:25.699 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37927.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42121.log 2026-03-24T02:52:25.701 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88178.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.88178.log.gz 2026-03-24T02:52:25.702 INFO:teuthology.orchestra.run.vm06.stderr: 91.1% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2026-03-24T02:52:25.704 INFO:teuthology.orchestra.run.vm07.stderr: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.37927.log.gz 2026-03-24T02:52:25.710 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77289.log 2026-03-24T02:52:25.711 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42121.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42121.log.gz 2026-03-24T02:52:25.722 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80490.log 2026-03-24T02:52:25.723 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.77289.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.77289.log.gz 2026-03-24T02:52:25.723 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46062.log 2026-03-24T02:52:25.730 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80490.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80490.log.gz 2026-03-24T02:52:25.730 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89525.log 2026-03-24T02:52:25.734 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46062.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46062.log.gz 2026-03-24T02:52:25.734 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75038.log 2026-03-24T02:52:25.735 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.89525.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.89525.log.gz 2026-03-24T02:52:25.740 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48427.log.gz 2026-03-24T02:52:25.746 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81162.log 2026-03-24T02:52:25.747 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.75038.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75038.log.gz 2026-03-24T02:52:25.755 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85051.log 2026-03-24T02:52:25.756 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81162.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81162.log.gz 2026-03-24T02:52:25.758 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75299.log 2026-03-24T02:52:25.766 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85051.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.85051.log.gz 2026-03-24T02:52:25.770 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49189.log.gz 2026-03-24T02:52:25.770 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60569.log 2026-03-24T02:52:25.771 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.75299.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75299.log.gz 2026-03-24T02:52:25.771 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84061.log 2026-03-24T02:52:25.771 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.60569.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60569.log.gz 2026-03-24T02:52:25.772 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77835.log 2026-03-24T02:52:25.772 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84061.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84061.log.gz 2026-03-24T02:52:25.772 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83318.log 2026-03-24T02:52:25.772 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77835.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41633.log.gz 2026-03-24T02:52:25.773 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86645.log 2026-03-24T02:52:25.773 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72525.log 2026-03-24T02:52:25.773 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83318.log: /var/log/ceph/cluster1-client.mirror.86645.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83476.log 2026-03-24T02:52:25.773 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42697.log 2026-03-24T02:52:25.774 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.72525.log: /var/log/ceph/cluster1-client.mirror.83476.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42298.log 2026-03-24T02:52:25.775 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42697.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77835.log.gz 2026-03-24T02:52:25.775 INFO:teuthology.orchestra.run.vm07.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83476.log.gz 2026-03-24T02:52:25.775 INFO:teuthology.orchestra.run.vm07.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83318.log.gz 2026-03-24T02:52:25.775 INFO:teuthology.orchestra.run.vm07.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.86645.log.gz 2026-03-24T02:52:25.775 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64473.log 2026-03-24T02:52:25.775 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42298.log: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.72525.log.gz 2026-03-24T02:52:25.775 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60356.log 2026-03-24T02:52:25.776 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59613.log 2026-03-24T02:52:25.776 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.60356.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60356.log.gz 2026-03-24T02:52:25.776 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78371.log 2026-03-24T02:52:25.777 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59613.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.59613.log.gz 2026-03-24T02:52:25.777 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83836.log 2026-03-24T02:52:25.777 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78371.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.78371.log.gz 2026-03-24T02:52:25.778 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75239.log 2026-03-24T02:52:25.778 INFO:teuthology.orchestra.run.vm07.stderr: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.42298.log.gz 2026-03-24T02:52:25.778 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83836.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.83836.log.gz 2026-03-24T02:52:25.778 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64473.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89162.log 2026-03-24T02:52:25.779 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64473.log.gz 2026-03-24T02:52:25.779 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.75239.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.42697.log.gz 2026-03-24T02:52:25.782 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25754.log 2026-03-24T02:52:25.782 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.89162.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.89162.log.gz 2026-03-24T02:52:25.783 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30892.log 2026-03-24T02:52:25.783 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.25754.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25754.log.gz 2026-03-24T02:52:25.783 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87122.log 2026-03-24T02:52:25.784 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30892.log: 97.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75239.log.gz 2026-03-24T02:52:25.784 INFO:teuthology.orchestra.run.vm07.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30892.log.gz 2026-03-24T02:52:25.784 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26428.log 2026-03-24T02:52:25.784 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87122.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69083.log 2026-03-24T02:52:25.785 INFO:teuthology.orchestra.run.vm07.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.87122.log.gz 2026-03-24T02:52:25.785 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.26428.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73675.log 2026-03-24T02:52:25.785 INFO:teuthology.orchestra.run.vm07.stderr: 77.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26428.log.gz 2026-03-24T02:52:25.785 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.69083.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43121.log 2026-03-24T02:52:25.786 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.73675.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69083.log.gz 2026-03-24T02:52:25.788 INFO:teuthology.orchestra.run.vm07.stderr: 96.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73675.log.gz 2026-03-24T02:52:25.788 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81652.log 2026-03-24T02:52:25.788 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43121.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45744.log 2026-03-24T02:52:25.790 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81652.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81652.log.gz 2026-03-24T02:52:25.793 INFO:teuthology.orchestra.run.vm07.stderr: 97.1% -- replaced with /var/log/ceph/cluster2-client.mirror.43121.log.gz 2026-03-24T02:52:25.793 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83039.log 2026-03-24T02:52:25.794 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45744.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30183.log 2026-03-24T02:52:25.794 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.83039.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45744.log.gz 2026-03-24T02:52:25.794 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26860.log 2026-03-24T02:52:25.795 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30183.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.83039.log.gz 2026-03-24T02:52:25.795 INFO:teuthology.orchestra.run.vm07.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.30183.log.gz 2026-03-24T02:52:25.795 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77212.log 2026-03-24T02:52:25.795 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.26860.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82055.log 2026-03-24T02:52:25.796 INFO:teuthology.orchestra.run.vm07.stderr: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.26860.log.gz 2026-03-24T02:52:25.796 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77212.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71753.log 2026-03-24T02:52:25.796 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77212.log.gz 2026-03-24T02:52:25.796 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82055.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.82055.log.gz 2026-03-24T02:52:25.797 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48729.log 2026-03-24T02:52:25.797 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.71753.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69655.log 2026-03-24T02:52:25.798 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.48729.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.71753.log.gz 2026-03-24T02:52:25.798 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42584.log 2026-03-24T02:52:25.799 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.69655.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.69655.log.gz 2026-03-24T02:52:25.800 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78464.log 2026-03-24T02:52:25.800 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.42584.log: 84.8% -- replaced with /var/log/ceph/cluster1-client.mirror.42584.log.gz 2026-03-24T02:52:25.800 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76519.log 2026-03-24T02:52:25.801 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78464.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.78464.log.gz 2026-03-24T02:52:25.801 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61507.log 2026-03-24T02:52:25.805 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76519.log: 96.8% -- replaced with /var/log/ceph/cluster2-client.mirror.76519.log.gz 2026-03-24T02:52:25.805 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74203.log 2026-03-24T02:52:25.805 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61507.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.61507.log.gz 2026-03-24T02:52:25.805 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88097.log 2026-03-24T02:52:25.806 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.74203.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.74203.log.gz 2026-03-24T02:52:25.806 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43605.log 2026-03-24T02:52:25.807 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88097.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88097.log.gz 2026-03-24T02:52:25.807 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41507.log 2026-03-24T02:52:25.807 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43605.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.43605.log.gz 2026-03-24T02:52:25.808 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46224.log 2026-03-24T02:52:25.808 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.41507.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71328.log 2026-03-24T02:52:25.811 INFO:teuthology.orchestra.run.vm07.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.41507.log.gz 2026-03-24T02:52:25.811 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46224.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76611.log 2026-03-24T02:52:25.823 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.71328.log: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.71328.log.gz 2026-03-24T02:52:25.831 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52829.log 2026-03-24T02:52:25.831 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76611.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76611.log.gz 2026-03-24T02:52:25.831 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78250.log 2026-03-24T02:52:25.838 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.52829.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52829.log.gz 2026-03-24T02:52:25.838 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54351.log 2026-03-24T02:52:25.847 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78250.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.78250.log.gz 2026-03-24T02:52:25.847 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71350.log 2026-03-24T02:52:25.847 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.54351.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26911.log 2026-03-24T02:52:25.848 INFO:teuthology.orchestra.run.vm07.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.54351.log.gz 2026-03-24T02:52:25.852 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71350.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34688.log 2026-03-24T02:52:25.852 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81375.log 2026-03-24T02:52:25.857 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.26911.log: 97.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71350.log.gz 2026-03-24T02:52:25.862 INFO:teuthology.orchestra.run.vm07.stderr: 68.4% -- replaced with /var/log/ceph/cluster1-client.mirror.26911.log.gz 2026-03-24T02:52:25.863 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34688.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.34688.log.gz 2026-03-24T02:52:25.864 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61273.log 2026-03-24T02:52:25.865 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81375.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.81375.log.gz 2026-03-24T02:52:25.865 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35570.log 2026-03-24T02:52:25.866 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61273.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.61273.log.gz 2026-03-24T02:52:25.868 INFO:teuthology.orchestra.run.vm07.stderr: 96.1% -- replaced with /var/log/ceph/cluster2-client.mirror.48729.log.gz 2026-03-24T02:52:25.877 INFO:teuthology.orchestra.run.vm07.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.48513.log.gz 2026-03-24T02:52:25.878 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59170.log 2026-03-24T02:52:25.878 INFO:teuthology.orchestra.run.vm07.stderr:gzip/var/log/ceph/cluster2-client.mirror.35570.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82075.log 2026-03-24T02:52:25.879 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59170.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.35570.log.gz 2026-03-24T02:52:25.879 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.59170.log.gz 2026-03-24T02:52:25.882 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82609.log 2026-03-24T02:52:25.882 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82075.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83363.log 2026-03-24T02:52:25.883 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82609.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.82609.log.gz 2026-03-24T02:52:25.883 INFO:teuthology.orchestra.run.vm07.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82075.log.gz 2026-03-24T02:52:25.886 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.23028.log 2026-03-24T02:52:25.887 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83363.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73992.log 2026-03-24T02:52:25.887 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.23028.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.83363.log.gz 2026-03-24T02:52:25.887 INFO:teuthology.orchestra.run.vm07.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.23028.log.gz 2026-03-24T02:52:25.887 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46879.log 2026-03-24T02:52:25.888 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.73992.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35849.log 2026-03-24T02:52:25.888 INFO:teuthology.orchestra.run.vm07.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.73992.log.gz 2026-03-24T02:52:25.890 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46879.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47308.log 2026-03-24T02:52:25.891 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35849.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.35849.log.gz 2026-03-24T02:52:25.891 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36256.log 2026-03-24T02:52:25.891 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47308.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48113.log 2026-03-24T02:52:25.892 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36256.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47308.log.gz 2026-03-24T02:52:25.892 INFO:teuthology.orchestra.run.vm07.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36256.log.gz 2026-03-24T02:52:25.892 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82274.log 2026-03-24T02:52:25.893 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.48113.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48113.log.gz 2026-03-24T02:52:25.893 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78158.log 2026-03-24T02:52:25.894 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82274.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.82274.log.gz 2026-03-24T02:52:25.894 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57903.log 2026-03-24T02:52:25.894 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78158.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.78158.log.gz 2026-03-24T02:52:25.895 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42439.log 2026-03-24T02:52:25.895 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57903.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57903.log.gz 2026-03-24T02:52:25.895 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31764.log 2026-03-24T02:52:25.896 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.42439.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44084.log 2026-03-24T02:52:25.896 INFO:teuthology.orchestra.run.vm07.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.42439.log.gz 2026-03-24T02:52:25.897 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31764.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64276.log 2026-03-24T02:52:25.897 INFO:teuthology.orchestra.run.vm07.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31764.log.gz 2026-03-24T02:52:25.897 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44084.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75197.log 2026-03-24T02:52:25.900 INFO:teuthology.orchestra.run.vm07.stderr: 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.44084.log.gz 2026-03-24T02:52:25.900 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64276.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64276.log.gz 2026-03-24T02:52:25.901 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63665.log 2026-03-24T02:52:25.902 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.75197.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75197.log.gz 2026-03-24T02:52:25.902 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82175.log 2026-03-24T02:52:25.903 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.63665.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.63665.log.gz 2026-03-24T02:52:25.903 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43258.log 2026-03-24T02:52:25.904 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82175.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.82175.log.gz 2026-03-24T02:52:25.904 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57646.log 2026-03-24T02:52:25.905 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43258.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44950.log 2026-03-24T02:52:25.905 INFO:teuthology.orchestra.run.vm07.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.43258.log.gz 2026-03-24T02:52:25.905 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57646.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86668.log 2026-03-24T02:52:25.906 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.57646.log.gz 2026-03-24T02:52:25.906 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44950.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44950.log.gz 2026-03-24T02:52:25.907 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55950.log 2026-03-24T02:52:25.907 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.86668.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.86668.log.gz 2026-03-24T02:52:25.907 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69873.log 2026-03-24T02:52:25.908 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55950.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26269.log 2026-03-24T02:52:25.908 INFO:teuthology.orchestra.run.vm07.stderr: 96.0% -- replaced with /var/log/ceph/cluster2-client.mirror.46224.log.gz 2026-03-24T02:52:25.908 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55950.log.gz 2026-03-24T02:52:25.908 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.51374.log 2026-03-24T02:52:25.908 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.69873.log: /var/log/ceph/cluster1-client.admin.26269.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26269.log.gz 2026-03-24T02:52:25.909 INFO:teuthology.orchestra.run.vm07.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69873.log.gz 2026-03-24T02:52:25.909 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84421.log 2026-03-24T02:52:25.909 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.39811.log 2026-03-24T02:52:25.909 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.2.51374.log: /var/log/ceph/cluster1-client.mirror.84421.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45508.log 2026-03-24T02:52:25.909 INFO:teuthology.orchestra.run.vm07.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84421.log.gz 2026-03-24T02:52:25.910 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.0.39811.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40335.log 2026-03-24T02:52:25.912 INFO:teuthology.orchestra.run.vm07.stderr: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.0.39811.log.gz 2026-03-24T02:52:25.915 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45508.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.2.51374.log.gz 2026-03-24T02:52:25.915 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43647.log 2026-03-24T02:52:25.916 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40335.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65777.log 2026-03-24T02:52:25.916 INFO:teuthology.orchestra.run.vm07.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40335.log.gz 2026-03-24T02:52:25.917 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43647.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.43647.log.gz 2026-03-24T02:52:25.917 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87815.log 2026-03-24T02:52:25.917 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.65777.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.65777.log.gz 2026-03-24T02:52:25.918 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61103.log 2026-03-24T02:52:25.918 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87815.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.87815.log.gz 2026-03-24T02:52:25.918 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41079.log 2026-03-24T02:52:25.919 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61103.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.61103.log.gz 2026-03-24T02:52:25.919 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40021.log 2026-03-24T02:52:25.919 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41079.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41079.log.gz 2026-03-24T02:52:25.920 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57004.log 2026-03-24T02:52:25.920 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40021.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.40021.log.gz 2026-03-24T02:52:25.920 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82751.log 2026-03-24T02:52:25.921 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57004.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57004.log.gz 2026-03-24T02:52:25.921 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83813.log 2026-03-24T02:52:25.922 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82751.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.82751.log.gz 2026-03-24T02:52:25.922 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29421.log 2026-03-24T02:52:25.922 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83813.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83813.log.gz 2026-03-24T02:52:25.923 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34919.log 2026-03-24T02:52:25.923 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.29421.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42255.log 2026-03-24T02:52:25.926 INFO:teuthology.orchestra.run.vm07.stderr: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.29421.log.gz 2026-03-24T02:52:25.926 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34919.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.34919.log.gz 2026-03-24T02:52:25.926 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88383.log 2026-03-24T02:52:25.927 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42255.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76194.log 2026-03-24T02:52:25.929 INFO:teuthology.orchestra.run.vm07.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.42255.log.gz 2026-03-24T02:52:25.930 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88383.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88383.log.gz 2026-03-24T02:52:25.930 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53192.log 2026-03-24T02:52:25.931 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76194.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76194.log.gz 2026-03-24T02:52:25.931 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84398.log 2026-03-24T02:52:25.931 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.53192.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52592.log 2026-03-24T02:52:25.932 INFO:teuthology.orchestra.run.vm07.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.53192.log.gz 2026-03-24T02:52:25.932 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84398.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84398.log.gz 2026-03-24T02:52:25.932 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74692.log 2026-03-24T02:52:25.933 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.52592.log: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52592.log.gz 2026-03-24T02:52:25.933 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71308.log 2026-03-24T02:52:25.938 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.74692.log: 97.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74692.log.gz 2026-03-24T02:52:25.938 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66653.log 2026-03-24T02:52:25.939 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71308.log: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71308.log.gz 2026-03-24T02:52:25.940 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73653.log 2026-03-24T02:52:25.940 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66653.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66653.log.gz 2026-03-24T02:52:25.940 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83588.log 2026-03-24T02:52:25.941 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.73653.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.73653.log.gz 2026-03-24T02:52:25.942 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63807.log 2026-03-24T02:52:25.942 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83588.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83588.log.gz 2026-03-24T02:52:25.942 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86161.log 2026-03-24T02:52:25.943 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.63807.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63807.log.gz 2026-03-24T02:52:25.943 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39640.log 2026-03-24T02:52:25.943 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86161.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.86161.log.gz 2026-03-24T02:52:25.944 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46778.log 2026-03-24T02:52:25.944 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39640.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67201.log 2026-03-24T02:52:25.969 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46778.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54218.log 2026-03-24T02:52:25.974 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.67201.log: 88.3%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26894.log 2026-03-24T02:52:25.974 INFO:teuthology.orchestra.run.vm07.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.67201.log.gz 2026-03-24T02:52:25.983 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.54218.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.54218.log.gz 2026-03-24T02:52:25.987 INFO:teuthology.orchestra.run.vm07.stderr: 96.0% -- replaced with /var/log/ceph/cluster2-client.mirror.45508.log.gz 2026-03-24T02:52:25.998 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86250.log 2026-03-24T02:52:25.999 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.26894.log: 64.8% -- replaced with /var/log/ceph/cluster1-client.mirror.26894.log.gz 2026-03-24T02:52:25.999 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43293.log 2026-03-24T02:52:26.006 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.86250.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.86250.log.gz 2026-03-24T02:52:26.006 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30985.log 2026-03-24T02:52:26.007 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43293.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.43293.log.gz 2026-03-24T02:52:26.007 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75819.log 2026-03-24T02:52:26.014 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30985.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30985.log.gz 2026-03-24T02:52:26.014 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38199.log 2026-03-24T02:52:26.023 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.75819.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.75819.log.gz 2026-03-24T02:52:26.023 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84263.log 2026-03-24T02:52:26.030 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38199.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38199.log.gz 2026-03-24T02:52:26.031 INFO:teuthology.orchestra.run.vm07.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46879.log.gz 2026-03-24T02:52:26.034 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74435.log 2026-03-24T02:52:26.035 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84263.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84263.log.gz 2026-03-24T02:52:26.038 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66161.log 2026-03-24T02:52:26.039 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.74435.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.74435.log.gz 2026-03-24T02:52:26.042 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38057.log 2026-03-24T02:52:26.043 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66161.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66161.log.gz 2026-03-24T02:52:26.046 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26877.log 2026-03-24T02:52:26.047 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.38057.log: 88.9% -- replaced with /var/log/ceph/cluster1-client.admin.38057.log.gz 2026-03-24T02:52:26.047 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26528.log 2026-03-24T02:52:26.047 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.26877.log: 68.1% -- replaced with /var/log/ceph/cluster1-client.mirror.26877.log.gz 2026-03-24T02:52:26.048 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66882.log 2026-03-24T02:52:26.048 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.26528.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54319.log 2026-03-24T02:52:26.051 INFO:teuthology.orchestra.run.vm07.stderr: 94.7% -- replaced with /var/log/ceph/cluster1-client.mirror.26528.log.gz 2026-03-24T02:52:26.052 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66882.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64530.log 2026-03-24T02:52:26.052 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54319.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.66882.log.gz 2026-03-24T02:52:26.053 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54319.log.gz 2026-03-24T02:52:26.053 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53876.log 2026-03-24T02:52:26.054 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64530.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.64530.log.gz 2026-03-24T02:52:26.054 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.26427.log 2026-03-24T02:52:26.055 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53876.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53876.log.gz 2026-03-24T02:52:26.055 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49466.log 2026-03-24T02:52:26.055 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.0.26427.log: 78.3% -- replaced with /var/log/ceph/cluster1-client.mirror.0.26427.log.gz 2026-03-24T02:52:26.056 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41275.log 2026-03-24T02:52:26.066 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.49466.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27460.log 2026-03-24T02:52:26.066 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41275.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41275.log.gz 2026-03-24T02:52:26.090 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66218.log 2026-03-24T02:52:26.091 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.27460.log: 54.2% -- replaced with /var/log/ceph/cluster2-client.mirror.27460.log.gz 2026-03-24T02:52:26.091 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48871.log 2026-03-24T02:52:26.092 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66218.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.66218.log.gz 2026-03-24T02:52:26.094 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46778.log.gz 2026-03-24T02:52:26.102 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34611.log 2026-03-24T02:52:26.110 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.48871.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57085.log 2026-03-24T02:52:26.112 INFO:teuthology.orchestra.run.vm06.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2026-03-24T02:52:26.114 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34611.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.34611.log.gz 2026-03-24T02:52:26.118 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83993.log 2026-03-24T02:52:26.122 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57085.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.57085.log.gz 2026-03-24T02:52:26.130 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49990.log 2026-03-24T02:52:26.131 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83993.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.83993.log.gz 2026-03-24T02:52:26.131 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65983.log 2026-03-24T02:52:26.138 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.49990.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49990.log.gz 2026-03-24T02:52:26.138 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43740.log 2026-03-24T02:52:26.142 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65983.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65983.log.gz 2026-03-24T02:52:26.150 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31113.log 2026-03-24T02:52:26.153 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43740.log: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.43740.log.gz 2026-03-24T02:52:26.155 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.1 has been restored 2026-03-24T02:52:26.158 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75520.log 2026-03-24T02:52:26.159 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31113.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.31113.log.gz 2026-03-24T02:52:26.166 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81610.log 2026-03-24T02:52:26.167 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.75520.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75520.log.gz 2026-03-24T02:52:26.174 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85903.log 2026-03-24T02:52:26.175 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81610.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.81610.log.gz 2026-03-24T02:52:26.175 INFO:teuthology.orchestra.run.vm07.stderr: 95.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88138.log 2026-03-24T02:52:26.175 INFO:teuthology.orchestra.run.vm07.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.48871.log.gz 2026-03-24T02:52:26.176 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.85903.log: 86.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85903.log.gz 2026-03-24T02:52:26.176 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65844.log 2026-03-24T02:52:26.177 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88138.log: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88138.log.gz 2026-03-24T02:52:26.178 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29776.log 2026-03-24T02:52:26.179 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65844.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65844.log.gz 2026-03-24T02:52:26.180 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42230.log 2026-03-24T02:52:26.180 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29776.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29776.log.gz 2026-03-24T02:52:26.180 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81510.log 2026-03-24T02:52:26.181 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.42230.log: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.42230.log.gz 2026-03-24T02:52:26.181 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56109.log 2026-03-24T02:52:26.183 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81510.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81510.log.gz 2026-03-24T02:52:26.183 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58290.log 2026-03-24T02:52:26.183 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56109.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56109.log.gz 2026-03-24T02:52:26.184 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72505.log 2026-03-24T02:52:26.184 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.58290.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71164.log 2026-03-24T02:52:26.185 INFO:teuthology.orchestra.run.vm07.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.58290.log.gz 2026-03-24T02:52:26.186 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72505.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72505.log.gz 2026-03-24T02:52:26.187 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33726.log 2026-03-24T02:52:26.191 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71164.log: 97.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71164.log.gz 2026-03-24T02:52:26.191 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83566.log 2026-03-24T02:52:26.192 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33726.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33726.log.gz 2026-03-24T02:52:26.192 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.28312.log 2026-03-24T02:52:26.193 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83566.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83566.log.gz 2026-03-24T02:52:26.193 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86689.log 2026-03-24T02:52:26.193 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.28312.log: 78.0% -- replaced with /var/log/ceph/cluster1-client.mirror.28312.log.gz 2026-03-24T02:52:26.194 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45767.log 2026-03-24T02:52:26.194 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86689.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.86689.log.gz 2026-03-24T02:52:26.194 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26534.log 2026-03-24T02:52:26.202 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45767.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44349.log 2026-03-24T02:52:26.204 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.26534.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49466.log.gz 2026-03-24T02:52:26.205 INFO:teuthology.orchestra.run.vm07.stderr: 85.2% -- replaced with /var/log/ceph/cluster1-client.mirror.26534.log.gz 2026-03-24T02:52:26.210 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42408.log 2026-03-24T02:52:26.211 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44349.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31950.log 2026-03-24T02:52:26.213 INFO:teuthology.orchestra.run.vm07.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44349.log.gz 2026-03-24T02:52:26.214 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.42408.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.admin.42408.log.gz 2026-03-24T02:52:26.222 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81752.log 2026-03-24T02:52:26.223 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31950.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31950.log.gz 2026-03-24T02:52:26.223 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63297.log 2026-03-24T02:52:26.223 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81752.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81752.log.gz 2026-03-24T02:52:26.224 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44213.log 2026-03-24T02:52:26.224 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.63297.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63297.log.gz 2026-03-24T02:52:26.224 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83701.log 2026-03-24T02:52:26.225 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44213.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68361.log 2026-03-24T02:52:26.227 INFO:teuthology.orchestra.run.vm07.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44213.log.gz 2026-03-24T02:52:26.228 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83701.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.83701.log.gz 2026-03-24T02:52:26.234 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34317.log 2026-03-24T02:52:26.235 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68361.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.68361.log.gz 2026-03-24T02:52:26.235 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63467.log 2026-03-24T02:52:26.235 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34317.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44564.log 2026-03-24T02:52:26.237 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.34317.log.gz 2026-03-24T02:52:26.238 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.63467.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63467.log.gz 2026-03-24T02:52:26.238 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80700.log 2026-03-24T02:52:26.238 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44564.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70383.log 2026-03-24T02:52:26.241 INFO:teuthology.orchestra.run.vm07.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44564.log.gz 2026-03-24T02:52:26.242 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80700.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.80700.log.gz 2026-03-24T02:52:26.242 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54106.log 2026-03-24T02:52:26.246 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.70383.log: 96.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70383.log.gz 2026-03-24T02:52:26.246 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82246.log 2026-03-24T02:52:26.246 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54106.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54106.log.gz 2026-03-24T02:52:26.247 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61707.log 2026-03-24T02:52:26.247 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82246.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.82246.log.gz 2026-03-24T02:52:26.247 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68984.log 2026-03-24T02:52:26.248 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61707.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74470.log 2026-03-24T02:52:26.252 INFO:teuthology.orchestra.run.vm07.stderr: 97.1% -- replaced with /var/log/ceph/cluster2-client.mirror.61707.log.gz 2026-03-24T02:52:26.253 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68984.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.68984.log.gz 2026-03-24T02:52:26.253 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29613.log 2026-03-24T02:52:26.253 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.74470.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.74470.log.gz 2026-03-24T02:52:26.254 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38827.log 2026-03-24T02:52:26.254 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.29613.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.29613.log.gz 2026-03-24T02:52:26.254 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48404.log 2026-03-24T02:52:26.255 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.38827.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.admin.38827.log.gz 2026-03-24T02:52:26.255 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44498.log 2026-03-24T02:52:26.256 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.48404.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48404.log.gz 2026-03-24T02:52:26.256 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63196.log 2026-03-24T02:52:26.256 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.44498.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84443.log 2026-03-24T02:52:26.259 INFO:teuthology.orchestra.run.vm07.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.44498.log.gz 2026-03-24T02:52:26.260 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.63196.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63196.log.gz 2026-03-24T02:52:26.260 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80884.log 2026-03-24T02:52:26.260 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84443.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84443.log.gz 2026-03-24T02:52:26.260 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71733.log 2026-03-24T02:52:26.261 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80884.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80884.log.gz 2026-03-24T02:52:26.261 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66193.log 2026-03-24T02:52:26.262 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71733.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.71733.log.gz 2026-03-24T02:52:26.263 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.26490.log 2026-03-24T02:52:26.263 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.66193.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.66193.log.gz 2026-03-24T02:52:26.263 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69186.log 2026-03-24T02:52:26.264 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.0.26490.log: 74.0% -- replaced with /var/log/ceph/cluster2-client.mirror.0.26490.log.gz 2026-03-24T02:52:26.264 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31636.log 2026-03-24T02:52:26.265 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.69186.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.69186.log.gz 2026-03-24T02:52:26.265 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35215.log 2026-03-24T02:52:26.265 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31636.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31636.log.gz 2026-03-24T02:52:26.265 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62352.log 2026-03-24T02:52:26.266 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.35215.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.35215.log.gz 2026-03-24T02:52:26.266 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89043.log 2026-03-24T02:52:26.267 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62352.log: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.62352.log.gz 2026-03-24T02:52:26.268 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42867.log 2026-03-24T02:52:26.268 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.89043.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89043.log.gz 2026-03-24T02:52:26.268 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76704.log 2026-03-24T02:52:26.269 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42867.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.42867.log.gz 2026-03-24T02:52:26.269 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26962.log 2026-03-24T02:52:26.269 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76704.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76704.log.gz 2026-03-24T02:52:26.270 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35031.log 2026-03-24T02:52:26.270 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.26962.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.26962.log.gz 2026-03-24T02:52:26.270 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34553.log 2026-03-24T02:52:26.271 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.35031.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71892.log 2026-03-24T02:52:26.273 INFO:teuthology.orchestra.run.vm07.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.45767.log.gz 2026-03-24T02:52:26.273 INFO:teuthology.orchestra.run.vm07.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.35031.log.gz 2026-03-24T02:52:26.274 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34553.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.34553.log.gz 2026-03-24T02:52:26.274 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75485.log 2026-03-24T02:52:26.275 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71892.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82365.log 2026-03-24T02:52:26.275 INFO:teuthology.orchestra.run.vm07.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.71892.log.gz 2026-03-24T02:52:26.275 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.75485.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87962.log 2026-03-24T02:52:26.275 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75485.log.gz 2026-03-24T02:52:26.276 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82365.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82365.log.gz 2026-03-24T02:52:26.276 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71927.log 2026-03-24T02:52:26.277 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.87962.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38858.log 2026-03-24T02:52:26.277 INFO:teuthology.orchestra.run.vm07.stderr: 81.4% -- replaced with /var/log/ceph/cluster1-client.mirror.87962.log.gz 2026-03-24T02:52:26.277 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71927.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37947.log 2026-03-24T02:52:26.277 INFO:teuthology.orchestra.run.vm07.stderr: 88.3%/var/log/ceph/cluster1-client.admin.38858.log: -- replaced with /var/log/ceph/cluster2-client.mirror.71927.log.gz 2026-03-24T02:52:26.278 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61188.log 2026-03-24T02:52:26.278 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.38858.log.gz 2026-03-24T02:52:26.278 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77539.log 2026-03-24T02:52:26.279 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61188.log: /var/log/ceph/cluster1-client.mirror.37947.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61188.log.gz 2026-03-24T02:52:26.279 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67748.log 2026-03-24T02:52:26.279 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.77539.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.77539.log.gz 2026-03-24T02:52:26.280 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82844.log 2026-03-24T02:52:26.280 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.67748.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.67748.log.gz 2026-03-24T02:52:26.281 INFO:teuthology.orchestra.run.vm07.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.37947.log.gz 2026-03-24T02:52:26.282 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51913.log 2026-03-24T02:52:26.282 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82844.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69041.log 2026-03-24T02:52:26.282 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82844.log.gz 2026-03-24T02:52:26.283 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.51913.log: 2.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72361.log 2026-03-24T02:52:26.283 INFO:teuthology.orchestra.run.vm07.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.51913.log.gz 2026-03-24T02:52:26.284 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.69041.log: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69041.log.gz 2026-03-24T02:52:26.284 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62175.log 2026-03-24T02:52:26.284 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72361.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30241.log 2026-03-24T02:52:26.284 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72361.log.gz 2026-03-24T02:52:26.285 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62175.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49775.log 2026-03-24T02:52:26.285 INFO:teuthology.orchestra.run.vm07.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62175.log.gz 2026-03-24T02:52:26.285 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30241.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30241.log.gz 2026-03-24T02:52:26.285 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81490.log 2026-03-24T02:52:26.286 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.49775.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55740.log 2026-03-24T02:52:26.286 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81490.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81490.log.gz 2026-03-24T02:52:26.298 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81703.log 2026-03-24T02:52:26.299 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.55740.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86818.log 2026-03-24T02:52:26.299 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81703.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.55740.log.gz 2026-03-24T02:52:26.299 INFO:teuthology.orchestra.run.vm07.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.81703.log.gz 2026-03-24T02:52:26.300 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89369.log 2026-03-24T02:52:26.303 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86818.log: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39640.log.gz 2026-03-24T02:52:26.306 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88826.log 2026-03-24T02:52:26.306 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.89369.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.89369.log.gz 2026-03-24T02:52:26.306 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.23106.log 2026-03-24T02:52:26.307 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.88826.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88826.log.gz 2026-03-24T02:52:26.307 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49045.log 2026-03-24T02:52:26.308 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.23106.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.23106.log.gz 2026-03-24T02:52:26.308 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39547.log 2026-03-24T02:52:26.318 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.49045.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61238.log 2026-03-24T02:52:26.321 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39547.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39547.log.gz 2026-03-24T02:52:26.326 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60453.log 2026-03-24T02:52:26.328 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61238.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.61238.log.gz 2026-03-24T02:52:26.338 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54705.log 2026-03-24T02:52:26.339 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.60453.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42318.log 2026-03-24T02:52:26.339 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.60453.log.gz 2026-03-24T02:52:26.346 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54705.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54052.log 2026-03-24T02:52:26.346 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54705.log.gz 2026-03-24T02:52:26.347 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.42318.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57580.log 2026-03-24T02:52:26.349 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54052.log: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.42318.log.gz 2026-03-24T02:52:26.354 INFO:teuthology.orchestra.run.vm07.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54052.log.gz 2026-03-24T02:52:26.358 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83611.log 2026-03-24T02:52:26.370 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57580.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.57580.log.gz 2026-03-24T02:52:26.370 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39139.log 2026-03-24T02:52:26.371 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83611.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83611.log.gz 2026-03-24T02:52:26.378 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73608.log 2026-03-24T02:52:26.383 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39139.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49775.log.gz 2026-03-24T02:52:26.384 INFO:teuthology.orchestra.run.vm07.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39139.log.gz 2026-03-24T02:52:26.384 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49571.log 2026-03-24T02:52:26.384 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.73608.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.73608.log.gz 2026-03-24T02:52:26.385 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56790.log 2026-03-24T02:52:26.385 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.49571.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37902.log 2026-03-24T02:52:26.398 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56790.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25594.log 2026-03-24T02:52:26.399 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.37902.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56790.log.gz 2026-03-24T02:52:26.399 INFO:teuthology.orchestra.run.vm07.stderr: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.37902.log.gz 2026-03-24T02:52:26.399 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49354.log 2026-03-24T02:52:26.399 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.25594.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25594.log.gz 2026-03-24T02:52:26.410 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42009.log 2026-03-24T02:52:26.411 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.49354.log: 94.6% -- replaced with /var/log/ceph/cluster1-client.mirror.49354.log.gz 2026-03-24T02:52:26.414 INFO:teuthology.orchestra.run.vm07.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49045.log.gz 2026-03-24T02:52:26.422 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40056.log 2026-03-24T02:52:26.423 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42009.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46023.log 2026-03-24T02:52:26.423 INFO:teuthology.orchestra.run.vm07.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.42009.log.gz 2026-03-24T02:52:26.424 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40056.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.85903.log 2026-03-24T02:52:26.424 INFO:teuthology.orchestra.run.vm07.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40056.log.gz 2026-03-24T02:52:26.430 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46023.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76646.log 2026-03-24T02:52:26.440 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.3.85903.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.3.85903.log.gz 2026-03-24T02:52:26.440 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27482.log 2026-03-24T02:52:26.441 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76646.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76646.log.gz 2026-03-24T02:52:26.450 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37244.log 2026-03-24T02:52:26.451 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.27482.log: 84.9% -- replaced with /var/log/ceph/cluster2-client.mirror.27482.log.gz 2026-03-24T02:52:26.451 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25702.log 2026-03-24T02:52:26.452 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37244.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37244.log.gz 2026-03-24T02:52:26.462 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79252.log 2026-03-24T02:52:26.463 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.25702.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25702.log.gz 2026-03-24T02:52:26.463 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36442.log 2026-03-24T02:52:26.464 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.79252.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.79252.log.gz 2026-03-24T02:52:26.464 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37058.log 2026-03-24T02:52:26.475 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36442.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.36442.log.gz 2026-03-24T02:52:26.475 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47800.log 2026-03-24T02:52:26.475 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37058.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26018.log 2026-03-24T02:52:26.475 INFO:teuthology.orchestra.run.vm07.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37058.log.gz 2026-03-24T02:52:26.482 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47800.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67968.log 2026-03-24T02:52:26.484 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86818.log.gz 2026-03-24T02:52:26.484 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54173.log 2026-03-24T02:52:26.484 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.67968.log: /var/log/ceph/cluster2-client.admin.26018.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.26018.log.gz 2026-03-24T02:52:26.485 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.67968.log.gz 2026-03-24T02:52:26.485 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87793.log 2026-03-24T02:52:26.486 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.54173.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54173.log.gz 2026-03-24T02:52:26.486 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75774.log 2026-03-24T02:52:26.486 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87793.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.87793.log.gz 2026-03-24T02:52:26.490 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34800.log 2026-03-24T02:52:26.498 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.75774.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.75774.log.gz 2026-03-24T02:52:26.502 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36035.log 2026-03-24T02:52:26.503 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34800.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34800.log.gz 2026-03-24T02:52:26.503 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47211.log 2026-03-24T02:52:26.510 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36035.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.36035.log.gz 2026-03-24T02:52:26.510 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62674.log 2026-03-24T02:52:26.518 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47211.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47211.log.gz 2026-03-24T02:52:26.518 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79060.log 2026-03-24T02:52:26.519 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62674.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62674.log.gz 2026-03-24T02:52:26.519 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84331.log 2026-03-24T02:52:26.520 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79060.log: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.46023.log.gz 2026-03-24T02:52:26.520 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26485.log 2026-03-24T02:52:26.521 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84331.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.79060.log.gz 2026-03-24T02:52:26.521 INFO:teuthology.orchestra.run.vm07.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84331.log.gz 2026-03-24T02:52:26.521 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77269.log 2026-03-24T02:52:26.522 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.26485.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.26485.log.gz 2026-03-24T02:52:26.530 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77002.log 2026-03-24T02:52:26.530 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77269.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44887.log 2026-03-24T02:52:26.531 INFO:teuthology.orchestra.run.vm07.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77269.log.gz 2026-03-24T02:52:26.532 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.77002.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.77002.log.gz 2026-03-24T02:52:26.535 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49571.log.gz 2026-03-24T02:52:26.542 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64934.log 2026-03-24T02:52:26.543 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44887.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81632.log 2026-03-24T02:52:26.543 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64934.log: 89.5% 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64934.log.gz 2026-03-24T02:52:26.543 INFO:teuthology.orchestra.run.vm07.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.44887.log.gz 2026-03-24T02:52:26.543 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88798.log 2026-03-24T02:52:26.544 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81632.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60650.log 2026-03-24T02:52:26.544 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88798.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81632.log.gz 2026-03-24T02:52:26.544 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45867.log 2026-03-24T02:52:26.545 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.60650.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.60650.log.gz 2026-03-24T02:52:26.545 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.27014.log 2026-03-24T02:52:26.546 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45867.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45867.log.gz 2026-03-24T02:52:26.546 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59578.log 2026-03-24T02:52:26.547 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.27014.log: 80.4% -- replaced with /var/log/ceph/cluster2-client.admin.27014.log.gz 2026-03-24T02:52:26.547 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53505.log 2026-03-24T02:52:26.547 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59578.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82935.log 2026-03-24T02:52:26.547 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.59578.log.gz 2026-03-24T02:52:26.548 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53505.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.53505.log.gz 2026-03-24T02:52:26.548 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72941.log 2026-03-24T02:52:26.550 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82935.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82935.log.gz 2026-03-24T02:52:26.550 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53604.log 2026-03-24T02:52:26.550 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72941.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72941.log.gz 2026-03-24T02:52:26.551 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48616.log 2026-03-24T02:52:26.551 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53604.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44712.log 2026-03-24T02:52:26.551 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53604.log.gz 2026-03-24T02:52:26.552 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.48616.log: 84.3% -- replaced with /var/log/ceph/cluster2-client.mirror.48616.log.gz 2026-03-24T02:52:26.552 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85497.log 2026-03-24T02:52:26.553 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44712.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44712.log.gz 2026-03-24T02:52:26.553 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47053.log 2026-03-24T02:52:26.554 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.85497.log: 64.1% -- replaced with /var/log/ceph/cluster2-client.mirror.85497.log.gz 2026-03-24T02:52:26.554 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76385.log 2026-03-24T02:52:26.554 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47053.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47053.log.gz 2026-03-24T02:52:26.555 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65540.log 2026-03-24T02:52:26.555 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76385.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76385.log.gz 2026-03-24T02:52:26.555 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57166.log 2026-03-24T02:52:26.556 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65540.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65540.log.gz 2026-03-24T02:52:26.556 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38643.log 2026-03-24T02:52:26.557 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57166.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42825.log 2026-03-24T02:52:26.557 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.57166.log.gz 2026-03-24T02:52:26.557 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.38643.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68080.log 2026-03-24T02:52:26.560 INFO:teuthology.orchestra.run.vm07.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.38643.log.gz 2026-03-24T02:52:26.561 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42825.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.42825.log.gz 2026-03-24T02:52:26.561 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81020.log 2026-03-24T02:52:26.562 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.68080.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.68080.log.gz 2026-03-24T02:52:26.562 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59512.log 2026-03-24T02:52:26.563 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81020.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81020.log.gz 2026-03-24T02:52:26.563 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31171.log 2026-03-24T02:52:26.564 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.59512.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44929.log 2026-03-24T02:52:26.565 INFO:teuthology.orchestra.run.vm07.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.59512.log.gz 2026-03-24T02:52:26.565 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31171.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31171.log.gz 2026-03-24T02:52:26.566 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42275.log 2026-03-24T02:52:26.567 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44929.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44929.log.gz 2026-03-24T02:52:26.567 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89499.log 2026-03-24T02:52:26.567 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.42275.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43932.log 2026-03-24T02:52:26.570 INFO:teuthology.orchestra.run.vm07.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.42275.log.gz 2026-03-24T02:52:26.570 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.89499.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.89499.log.gz 2026-03-24T02:52:26.571 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83453.log 2026-03-24T02:52:26.571 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.43932.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41444.log 2026-03-24T02:52:26.574 INFO:teuthology.orchestra.run.vm07.stderr: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43932.log.gz 2026-03-24T02:52:26.574 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83453.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85999.log 2026-03-24T02:52:26.575 INFO:teuthology.orchestra.run.vm07.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83453.log.gz 2026-03-24T02:52:26.575 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41444.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44306.log 2026-03-24T02:52:26.578 INFO:teuthology.orchestra.run.vm07.stderr: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.41444.log.gz 2026-03-24T02:52:26.578 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85999.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.85999.log.gz 2026-03-24T02:52:26.579 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86327.log 2026-03-24T02:52:26.579 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44306.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75799.log 2026-03-24T02:52:26.582 INFO:teuthology.orchestra.run.vm07.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44306.log.gz 2026-03-24T02:52:26.582 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.86327.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.86327.log.gz 2026-03-24T02:52:26.583 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73173.log 2026-03-24T02:52:26.584 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.75799.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75799.log.gz 2026-03-24T02:52:26.584 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73422.log 2026-03-24T02:52:26.587 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.73173.log: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73173.log.gz 2026-03-24T02:52:26.587 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66948.log 2026-03-24T02:52:26.588 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.73422.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73422.log.gz 2026-03-24T02:52:26.588 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73541.log 2026-03-24T02:52:26.589 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66948.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66948.log.gz 2026-03-24T02:52:26.589 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35977.log 2026-03-24T02:52:26.589 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.73541.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73541.log.gz 2026-03-24T02:52:26.590 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78336.log 2026-03-24T02:52:26.590 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35977.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35977.log.gz 2026-03-24T02:52:26.590 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67489.log 2026-03-24T02:52:26.591 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78336.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.78336.log.gz 2026-03-24T02:52:26.591 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87461.log 2026-03-24T02:52:26.592 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.67489.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67489.log.gz 2026-03-24T02:52:26.592 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86945.log 2026-03-24T02:52:26.592 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87461.log: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.87461.log.gz 2026-03-24T02:52:26.593 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.44778.log 2026-03-24T02:52:26.594 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86945.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.86945.log.gz 2026-03-24T02:52:26.594 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79559.log 2026-03-24T02:52:26.595 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.0.44778.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65442.log 2026-03-24T02:52:26.597 INFO:teuthology.orchestra.run.vm07.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.0.44778.log.gz 2026-03-24T02:52:26.597 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79559.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.79559.log.gz 2026-03-24T02:52:26.597 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36221.log 2026-03-24T02:52:26.598 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65442.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65442.log.gz 2026-03-24T02:52:26.598 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.80216.log 2026-03-24T02:52:26.599 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36221.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.36221.log.gz 2026-03-24T02:52:26.599 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69441.log 2026-03-24T02:52:26.600 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.1.80216.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45080.log 2026-03-24T02:52:26.600 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47800.log.gz 2026-03-24T02:52:26.609 INFO:teuthology.orchestra.run.vm07.stderr: 92.5%/var/log/ceph/cluster2-client.mirror.69441.log: -- replaced with /var/log/ceph/cluster1-client.mirror.1.80216.log.gz 2026-03-24T02:52:26.609 INFO:teuthology.orchestra.run.vm07.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.69441.log.gz 2026-03-24T02:52:26.609 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.57417.log 2026-03-24T02:52:26.610 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45080.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25940.log 2026-03-24T02:52:26.610 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.0.57417.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45080.log.gz 2026-03-24T02:52:26.611 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71142.log 2026-03-24T02:52:26.611 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.25940.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25940.log.gz 2026-03-24T02:52:26.618 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83948.log 2026-03-24T02:52:26.620 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.71142.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.71142.log.gz 2026-03-24T02:52:26.630 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39788.log 2026-03-24T02:52:26.631 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83948.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.83948.log.gz 2026-03-24T02:52:26.638 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63502.log 2026-03-24T02:52:26.650 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39788.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31450.log 2026-03-24T02:52:26.650 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.88798.log.gz 2026-03-24T02:52:26.650 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.63502.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63066.log 2026-03-24T02:52:26.651 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31450.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63502.log.gz 2026-03-24T02:52:26.651 INFO:teuthology.orchestra.run.vm07.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31450.log.gz 2026-03-24T02:52:26.651 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28763.log 2026-03-24T02:52:26.652 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.63066.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58615.log 2026-03-24T02:52:26.652 INFO:teuthology.orchestra.run.vm07.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.63066.log.gz 2026-03-24T02:52:26.653 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.28763.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75400.log 2026-03-24T02:52:26.653 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.28763.log.gz 2026-03-24T02:52:26.653 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.58615.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87256.log 2026-03-24T02:52:26.654 INFO:teuthology.orchestra.run.vm07.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.58615.log.gz 2026-03-24T02:52:26.654 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.75400.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75400.log.gz 2026-03-24T02:52:26.654 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31299.log 2026-03-24T02:52:26.655 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87256.log: 83.5% -- replaced with /var/log/ceph/cluster2-client.mirror.87256.log.gz 2026-03-24T02:52:26.655 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2026-03-24T02:52:26.655 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31299.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.31299.log.gz 2026-03-24T02:52:26.656 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35011.log 2026-03-24T02:52:26.656 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86277.log 2026-03-24T02:52:26.658 INFO:teuthology.orchestra.run.vm07.stderr: 92.8% -- replaced with /var/log/ceph/cluster2.audit.log.gz 2026-03-24T02:52:26.658 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35011.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81042.log 2026-03-24T02:52:26.661 INFO:teuthology.orchestra.run.vm07.stderr: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.35011.log.gz 2026-03-24T02:52:26.661 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.86277.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.86277.log.gz 2026-03-24T02:52:26.661 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77777.log 2026-03-24T02:52:26.662 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81042.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81042.log.gz 2026-03-24T02:52:26.662 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40614.log 2026-03-24T02:52:26.663 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77777.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77777.log.gz 2026-03-24T02:52:26.663 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70645.log 2026-03-24T02:52:26.663 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29448.log 2026-03-24T02:52:26.663 INFO:teuthology.orchestra.run.vm07.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.40614.log.gz 2026-03-24T02:52:26.664 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.70645.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70645.log.gz 2026-03-24T02:52:26.664 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82968.log 2026-03-24T02:52:26.665 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29448.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33354.log 2026-03-24T02:52:26.667 INFO:teuthology.orchestra.run.vm07.stderr: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.29448.log.gz 2026-03-24T02:52:26.668 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82968.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.82968.log.gz 2026-03-24T02:52:26.668 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79095.log 2026-03-24T02:52:26.668 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33354.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33354.log.gz 2026-03-24T02:52:26.669 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83903.log 2026-03-24T02:52:26.669 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79095.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79095.log.gz 2026-03-24T02:52:26.669 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.39640.log 2026-03-24T02:52:26.670 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83903.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.83903.log.gz 2026-03-24T02:52:26.670 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41487.log 2026-03-24T02:52:26.670 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.2.39640.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85163.log 2026-03-24T02:52:26.682 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41487.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38623.log 2026-03-24T02:52:26.685 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85163.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41487.log.gz 2026-03-24T02:52:26.685 INFO:teuthology.orchestra.run.vm07.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.85163.log.gz 2026-03-24T02:52:26.694 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81973.log 2026-03-24T02:52:26.697 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38623.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.38623.log.gz 2026-03-24T02:52:26.697 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.29663.log 2026-03-24T02:52:26.706 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81973.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81973.log.gz 2026-03-24T02:52:26.707 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61153.log 2026-03-24T02:52:26.721 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.1.29663.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.1.29663.log.gz 2026-03-24T02:52:26.721 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44478.log 2026-03-24T02:52:26.722 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61153.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.61153.log.gz 2026-03-24T02:52:26.734 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80176.log 2026-03-24T02:52:26.735 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44478.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30369.log 2026-03-24T02:52:26.737 INFO:teuthology.orchestra.run.vm07.stderr: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.44478.log.gz 2026-03-24T02:52:26.737 INFO:teuthology.orchestra.run.vm07.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39788.log.gz 2026-03-24T02:52:26.738 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80176.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62109.log 2026-03-24T02:52:26.738 INFO:teuthology.orchestra.run.vm07.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80176.log.gz 2026-03-24T02:52:26.738 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30369.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30369.log.gz 2026-03-24T02:52:26.739 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62372.log 2026-03-24T02:52:26.739 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62109.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62109.log.gz 2026-03-24T02:52:26.739 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38164.log 2026-03-24T02:52:26.740 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.62372.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.62372.log.gz 2026-03-24T02:52:26.741 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43056.log 2026-03-24T02:52:26.741 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38164.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.38164.log.gz 2026-03-24T02:52:26.742 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.28312.log 2026-03-24T02:52:26.742 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.43056.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29663.log 2026-03-24T02:52:26.745 INFO:teuthology.orchestra.run.vm07.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.43056.log.gz 2026-03-24T02:52:26.748 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.0.28312.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.0.28312.log.gz 2026-03-24T02:52:26.750 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34295.log 2026-03-24T02:52:26.751 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.29663.log: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.29663.log.gz 2026-03-24T02:52:26.751 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38934.log 2026-03-24T02:52:26.752 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.34295.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66444.log 2026-03-24T02:52:26.754 INFO:teuthology.orchestra.run.vm07.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.34295.log.gz 2026-03-24T02:52:26.755 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38934.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.38934.log.gz 2026-03-24T02:52:26.762 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50242.log 2026-03-24T02:52:26.763 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66444.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.66444.log.gz 2026-03-24T02:52:26.763 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49752.log 2026-03-24T02:52:26.765 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.50242.log: 95.2% -- replaced with /var/log/ceph/cluster1-client.mirror.50242.log.gz 2026-03-24T02:52:26.765 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68035.log 2026-03-24T02:52:26.766 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.49752.log: 84.3% -- replaced with /var/log/ceph/cluster2-client.mirror.49752.log.gz 2026-03-24T02:52:26.766 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74787.log 2026-03-24T02:52:26.767 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.68035.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.68035.log.gz 2026-03-24T02:52:26.767 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77406.log 2026-03-24T02:52:26.767 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.74787.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74787.log.gz 2026-03-24T02:52:26.768 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79967.log 2026-03-24T02:52:26.768 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77406.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77406.log.gz 2026-03-24T02:52:26.769 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86710.log 2026-03-24T02:52:26.769 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79967.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79967.log.gz 2026-03-24T02:52:26.769 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85812.log 2026-03-24T02:52:26.770 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86710.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.86710.log.gz 2026-03-24T02:52:26.770 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26758.log 2026-03-24T02:52:26.771 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85812.log: 86.1% -- replaced with /var/log/ceph/cluster1-client.mirror.85812.log.gz 2026-03-24T02:52:26.771 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36965.log 2026-03-24T02:52:26.771 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.26758.log: 80.4% -- replaced with /var/log/ceph/cluster1-client.admin.26758.log.gz 2026-03-24T02:52:26.772 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84916.log 2026-03-24T02:52:26.772 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36965.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.36965.log.gz 2026-03-24T02:52:26.772 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58938.log 2026-03-24T02:52:26.773 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84916.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84916.log.gz 2026-03-24T02:52:26.773 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88254.log 2026-03-24T02:52:26.774 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.58938.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.58938.log.gz 2026-03-24T02:52:26.774 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89112.log 2026-03-24T02:52:26.774 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.88254.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.88254.log.gz 2026-03-24T02:52:26.775 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61350.log 2026-03-24T02:52:26.775 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.89112.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.89112.log.gz 2026-03-24T02:52:26.775 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42732.log 2026-03-24T02:52:26.777 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.61350.log: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.61350.log.gz 2026-03-24T02:52:26.777 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66688.log 2026-03-24T02:52:26.777 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42732.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51282.log 2026-03-24T02:52:26.778 INFO:teuthology.orchestra.run.vm07.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.42732.log.gz 2026-03-24T02:52:26.778 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66688.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66688.log.gz 2026-03-24T02:52:26.778 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79731.log 2026-03-24T02:52:26.779 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.51282.log: 2.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51282.log.gz 2026-03-24T02:52:26.779 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32606.log 2026-03-24T02:52:26.780 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79731.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.79731.log.gz 2026-03-24T02:52:26.780 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59985.log 2026-03-24T02:52:26.781 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32606.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32606.log.gz 2026-03-24T02:52:26.781 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35243.log 2026-03-24T02:52:26.781 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59985.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.59985.log.gz 2026-03-24T02:52:26.782 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47722.log 2026-03-24T02:52:26.786 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35243.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30613.log 2026-03-24T02:52:26.814 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47722.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55915.log 2026-03-24T02:52:26.815 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30613.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.30613.log.gz 2026-03-24T02:52:26.822 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55673.log 2026-03-24T02:52:26.834 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55915.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.55915.log.gz 2026-03-24T02:52:26.834 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84218.log 2026-03-24T02:52:26.835 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55673.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55673.log.gz 2026-03-24T02:52:26.846 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80926.log 2026-03-24T02:52:26.847 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84218.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84218.log.gz 2026-03-24T02:52:26.854 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64311.log 2026-03-24T02:52:26.862 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80926.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58268.log 2026-03-24T02:52:26.863 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64311.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80926.log.gz 2026-03-24T02:52:26.867 INFO:teuthology.orchestra.run.vm07.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64311.log.gz 2026-03-24T02:52:26.878 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75841.log 2026-03-24T02:52:26.879 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.58268.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.58268.log.gz 2026-03-24T02:52:26.886 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.76008.log 2026-03-24T02:52:26.894 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.75841.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43760.log 2026-03-24T02:52:26.897 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.1.76008.log: 96.5% -- replaced with /var/log/ceph/cluster2-client.mirror.75841.log.gz 2026-03-24T02:52:26.906 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57681.log 2026-03-24T02:52:26.909 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.43760.log: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.43760.log.gz 2026-03-24T02:52:26.911 INFO:teuthology.orchestra.run.vm07.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.1.76008.log.gz 2026-03-24T02:52:26.922 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80038.log 2026-03-24T02:52:26.923 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57681.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.57681.log.gz 2026-03-24T02:52:26.930 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38600.log 2026-03-24T02:52:26.938 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80038.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88362.log 2026-03-24T02:52:26.942 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.38600.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80038.log.gz 2026-03-24T02:52:26.945 INFO:teuthology.orchestra.run.vm07.stderr: 94.1% -- replaced with /var/log/ceph/cluster1-client.mirror.0.57417.log.gz 2026-03-24T02:52:26.946 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39504.log 2026-03-24T02:52:26.946 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.88362.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88362.log.gz 2026-03-24T02:52:26.946 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89187.log 2026-03-24T02:52:26.947 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39504.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44279.log 2026-03-24T02:52:26.949 INFO:teuthology.orchestra.run.vm07.stderr: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.39504.log.gz 2026-03-24T02:52:26.950 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.89187.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.89187.log.gz 2026-03-24T02:52:26.950 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84533.log 2026-03-24T02:52:26.950 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.44279.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58833.log 2026-03-24T02:52:26.952 INFO:teuthology.orchestra.run.vm07.stderr: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.38600.log.gz 2026-03-24T02:52:26.953 INFO:teuthology.orchestra.run.vm07.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.44279.log.gz 2026-03-24T02:52:26.953 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84533.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.84533.log.gz 2026-03-24T02:52:26.953 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46108.log 2026-03-24T02:52:26.954 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.58833.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55785.log 2026-03-24T02:52:26.954 INFO:teuthology.orchestra.run.vm07.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58833.log.gz 2026-03-24T02:52:26.957 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.46108.log: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46108.log.gz 2026-03-24T02:52:26.957 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53432.log 2026-03-24T02:52:26.957 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.55785.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29582.log 2026-03-24T02:52:26.958 INFO:teuthology.orchestra.run.vm07.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55785.log.gz 2026-03-24T02:52:26.959 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53432.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53432.log.gz 2026-03-24T02:52:26.959 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46949.log 2026-03-24T02:52:26.959 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.29582.log: 88.7% -- replaced with /var/log/ceph/cluster1-client.admin.29582.log.gz 2026-03-24T02:52:26.959 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70680.log 2026-03-24T02:52:26.960 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46949.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26988.log 2026-03-24T02:52:26.971 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.70680.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70680.log.gz 2026-03-24T02:52:26.982 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31915.log 2026-03-24T02:52:26.983 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.26988.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.26988.log.gz 2026-03-24T02:52:26.983 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88731.log 2026-03-24T02:52:26.984 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31915.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.31915.log.gz 2026-03-24T02:52:26.984 INFO:teuthology.orchestra.run.vm07.stderr: 95.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88853.log 2026-03-24T02:52:26.984 INFO:teuthology.orchestra.run.vm07.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.35243.log.gz 2026-03-24T02:52:26.984 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78292.log 2026-03-24T02:52:26.985 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.88853.log: /var/log/ceph/cluster2-client.mirror.88731.log: 83.4% -- replaced with /var/log/ceph/cluster2-client.mirror.88731.log.gz 2026-03-24T02:52:26.985 INFO:teuthology.orchestra.run.vm07.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88853.log.gz 2026-03-24T02:52:26.985 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82773.log 2026-03-24T02:52:26.994 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78292.log: 97.0% -- replaced with /var/log/ceph/cluster2-client.mirror.78292.log.gz 2026-03-24T02:52:26.995 INFO:teuthology.orchestra.run.vm07.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.47722.log.gz 2026-03-24T02:52:26.998 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44777.log 2026-03-24T02:52:26.999 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82773.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82773.log.gz 2026-03-24T02:52:27.002 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44455.log 2026-03-24T02:52:27.003 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44777.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83791.log 2026-03-24T02:52:27.003 INFO:teuthology.orchestra.run.vm07.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44777.log.gz 2026-03-24T02:52:27.004 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.44455.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32699.log 2026-03-24T02:52:27.006 INFO:teuthology.orchestra.run.vm07.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.44455.log.gz 2026-03-24T02:52:27.007 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83791.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83791.log.gz 2026-03-24T02:52:27.007 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86422.log 2026-03-24T02:52:27.008 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32699.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32699.log.gz 2026-03-24T02:52:27.008 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34518.log 2026-03-24T02:52:27.009 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86422.log: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.86422.log.gz 2026-03-24T02:52:27.009 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.60096.log 2026-03-24T02:52:27.010 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34518.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.34518.log.gz 2026-03-24T02:52:27.010 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85582.log 2026-03-24T02:52:27.011 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.2.60096.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31822.log 2026-03-24T02:52:27.018 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40428.log 2026-03-24T02:52:27.019 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.85582.log: /var/log/ceph/cluster2-client.mirror.31822.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46949.log.gz 2026-03-24T02:52:27.019 INFO:teuthology.orchestra.run.vm07.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31822.log.gz 2026-03-24T02:52:27.030 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33691.log 2026-03-24T02:52:27.034 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40428.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40428.log.gz 2026-03-24T02:52:27.035 INFO:teuthology.orchestra.run.vm07.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.2.60096.log.gz 2026-03-24T02:52:27.035 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45451.log 2026-03-24T02:52:27.036 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33691.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33691.log.gz 2026-03-24T02:52:27.036 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48063.log 2026-03-24T02:52:27.036 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45451.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46353.log 2026-03-24T02:52:27.047 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.48063.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48063.log.gz 2026-03-24T02:52:27.058 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34765.log 2026-03-24T02:52:27.059 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46353.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37523.log 2026-03-24T02:52:27.066 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34765.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63231.log 2026-03-24T02:52:27.066 INFO:teuthology.orchestra.run.vm07.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.34765.log.gz 2026-03-24T02:52:27.070 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37523.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37523.log.gz 2026-03-24T02:52:27.070 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46623.log 2026-03-24T02:52:27.078 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.63231.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63231.log.gz 2026-03-24T02:52:27.078 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58223.log 2026-03-24T02:52:27.079 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46623.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46623.log.gz 2026-03-24T02:52:27.083 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80314.log 2026-03-24T02:52:27.086 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.58223.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.58223.log.gz 2026-03-24T02:52:27.086 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45390.log 2026-03-24T02:52:27.090 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80314.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.80314.log.gz 2026-03-24T02:52:27.098 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76970.log 2026-03-24T02:52:27.099 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45390.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80198.log 2026-03-24T02:52:27.106 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76970.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80153.log 2026-03-24T02:52:27.107 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80198.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76970.log.gz 2026-03-24T02:52:27.112 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80198.log.gz 2026-03-24T02:52:27.114 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45034.log 2026-03-24T02:52:27.122 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80153.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82793.log 2026-03-24T02:52:27.123 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45034.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80153.log.gz 2026-03-24T02:52:27.126 INFO:teuthology.orchestra.run.vm07.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.45034.log.gz 2026-03-24T02:52:27.134 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32796.log 2026-03-24T02:52:27.135 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82793.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82793.log.gz 2026-03-24T02:52:27.142 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68482.log 2026-03-24T02:52:27.144 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32796.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32796.log.gz 2026-03-24T02:52:27.150 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60749.log 2026-03-24T02:52:27.158 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68482.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.68482.log.gz 2026-03-24T02:52:27.166 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70329.log 2026-03-24T02:52:27.172 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.60749.log: 97.2% -- replaced with /var/log/ceph/cluster2-client.mirror.60749.log.gz 2026-03-24T02:52:27.174 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35054.log 2026-03-24T02:52:27.182 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.70329.log: 96.0% -- replaced with /var/log/ceph/cluster2-client.mirror.85582.log.gz 2026-03-24T02:52:27.182 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.67865.log 2026-03-24T02:52:27.183 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35054.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.70329.log.gz 2026-03-24T02:52:27.194 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70043.log 2026-03-24T02:52:27.196 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.0.67865.log: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.35054.log.gz 2026-03-24T02:52:27.204 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% 96.0% -- replaced with /var/log/ceph/cluster2-client.mirror.45451.log.gz 2026-03-24T02:52:27.204 INFO:teuthology.orchestra.run.vm07.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.45390.log.gz 2026-03-24T02:52:27.208 INFO:teuthology.orchestra.run.vm07.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.0.67865.log.gz 2026-03-24T02:52:27.208 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38665.log 2026-03-24T02:52:27.209 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.70043.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70043.log.gz 2026-03-24T02:52:27.209 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84286.log 2026-03-24T02:52:27.211 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38665.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72242.log 2026-03-24T02:52:27.213 INFO:teuthology.orchestra.run.vm07.stderr: 96.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38665.log.gz 2026-03-24T02:52:27.214 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84286.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47649.log 2026-03-24T02:52:27.214 INFO:teuthology.orchestra.run.vm07.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84286.log.gz 2026-03-24T02:52:27.215 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.72242.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.72242.log.gz 2026-03-24T02:52:27.215 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55623.log 2026-03-24T02:52:27.216 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47649.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47649.log.gz 2026-03-24T02:52:27.216 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37279.log 2026-03-24T02:52:27.217 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55623.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55623.log.gz 2026-03-24T02:52:27.217 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87691.log 2026-03-24T02:52:27.218 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37279.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.37279.log.gz 2026-03-24T02:52:27.218 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59706.log 2026-03-24T02:52:27.218 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.87691.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.87691.log.gz 2026-03-24T02:52:27.219 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49166.log 2026-03-24T02:52:27.219 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59706.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59706.log.gz 2026-03-24T02:52:27.220 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66126.log 2026-03-24T02:52:27.220 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.49166.log: 84.3% -- replaced with /var/log/ceph/cluster2-client.mirror.49166.log.gz 2026-03-24T02:52:27.221 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82436.log 2026-03-24T02:52:27.221 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66126.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66126.log.gz 2026-03-24T02:52:27.222 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77991.log 2026-03-24T02:52:27.223 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82436.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82436.log.gz 2026-03-24T02:52:27.223 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34409.log 2026-03-24T02:52:27.224 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.77991.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.77991.log.gz 2026-03-24T02:52:27.224 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69958.log 2026-03-24T02:52:27.225 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.34409.log: 88.9% -- replaced with /var/log/ceph/cluster1-client.admin.34409.log.gz 2026-03-24T02:52:27.225 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67033.log 2026-03-24T02:52:27.226 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.69958.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69958.log.gz 2026-03-24T02:52:27.226 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62900.log 2026-03-24T02:52:27.227 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.67033.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.67033.log.gz 2026-03-24T02:52:27.227 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37600.log 2026-03-24T02:52:27.228 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62900.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62900.log.gz 2026-03-24T02:52:27.228 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81255.log 2026-03-24T02:52:27.229 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37600.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.37600.log.gz 2026-03-24T02:52:27.229 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87420.log 2026-03-24T02:52:27.230 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81255.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81255.log.gz 2026-03-24T02:52:27.234 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81924.log 2026-03-24T02:52:27.235 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87420.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.87420.log.gz 2026-03-24T02:52:27.238 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86080.log 2026-03-24T02:52:27.239 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81924.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81924.log.gz 2026-03-24T02:52:27.242 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85096.log 2026-03-24T02:52:27.243 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86080.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.86080.log.gz 2026-03-24T02:52:27.243 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80539.log 2026-03-24T02:52:27.244 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85096.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.85096.log.gz 2026-03-24T02:52:27.244 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84241.log 2026-03-24T02:52:27.245 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80539.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.80539.log.gz 2026-03-24T02:52:27.245 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32136.log 2026-03-24T02:52:27.246 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84241.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84241.log.gz 2026-03-24T02:52:27.246 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28917.log 2026-03-24T02:52:27.247 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32136.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32136.log.gz 2026-03-24T02:52:27.247 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76935.log 2026-03-24T02:52:27.248 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.28917.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.28917.log.gz 2026-03-24T02:52:27.248 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.23141.log 2026-03-24T02:52:27.248 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76935.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76935.log.gz 2026-03-24T02:52:27.249 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83273.log 2026-03-24T02:52:27.249 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.23141.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.23141.log.gz 2026-03-24T02:52:27.249 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45938.log 2026-03-24T02:52:27.250 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83273.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83273.log.gz 2026-03-24T02:52:27.250 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79753.log 2026-03-24T02:52:27.251 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45938.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45938.log.gz 2026-03-24T02:52:27.251 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81275.log 2026-03-24T02:52:27.251 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79753.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79753.log.gz 2026-03-24T02:52:27.252 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50702.log 2026-03-24T02:52:27.253 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81275.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81275.log.gz 2026-03-24T02:52:27.253 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54376.log 2026-03-24T02:52:27.254 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.50702.log: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50702.log.gz 2026-03-24T02:52:27.254 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64024.log 2026-03-24T02:52:27.256 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54376.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54376.log.gz 2026-03-24T02:52:27.256 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86792.log 2026-03-24T02:52:27.256 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.64024.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26710.log 2026-03-24T02:52:27.257 INFO:teuthology.orchestra.run.vm07.stderr: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.64024.log.gz 2026-03-24T02:52:27.258 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86792.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.86792.log.gz 2026-03-24T02:52:27.258 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26928.log 2026-03-24T02:52:27.258 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.26710.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26710.log.gz 2026-03-24T02:52:27.259 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58798.log 2026-03-24T02:52:27.259 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.26928.log: 55.6% -- replaced with /var/log/ceph/cluster1-client.mirror.26928.log.gz 2026-03-24T02:52:27.259 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74248.log 2026-03-24T02:52:27.260 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.58798.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.58798.log.gz 2026-03-24T02:52:27.260 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88772.log 2026-03-24T02:52:27.261 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.74248.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.74248.log.gz 2026-03-24T02:52:27.262 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36128.log 2026-03-24T02:52:27.262 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88772.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88772.log.gz 2026-03-24T02:52:27.263 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74888.log 2026-03-24T02:52:27.263 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36128.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.36128.log.gz 2026-03-24T02:52:27.263 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40800.log 2026-03-24T02:52:27.264 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.74888.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74888.log.gz 2026-03-24T02:52:27.264 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84781.log 2026-03-24T02:52:27.265 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40800.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40800.log.gz 2026-03-24T02:52:27.265 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87668.log 2026-03-24T02:52:27.265 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84781.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84781.log.gz 2026-03-24T02:52:27.266 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62811.log 2026-03-24T02:52:27.266 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.87668.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.87668.log.gz 2026-03-24T02:52:27.267 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89137.log 2026-03-24T02:52:27.267 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62811.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62811.log.gz 2026-03-24T02:52:27.267 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70361.log 2026-03-24T02:52:27.268 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.89137.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.89137.log.gz 2026-03-24T02:52:27.268 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37712.log 2026-03-24T02:52:27.269 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.70361.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.70361.log.gz 2026-03-24T02:52:27.269 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31078.log 2026-03-24T02:52:27.270 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37712.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.37712.log.gz 2026-03-24T02:52:27.270 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31206.log 2026-03-24T02:52:27.271 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31078.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31078.log.gz 2026-03-24T02:52:27.271 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71251.log 2026-03-24T02:52:27.272 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31206.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31206.log.gz 2026-03-24T02:52:27.272 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87613.log 2026-03-24T02:52:27.272 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71251.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71251.log.gz 2026-03-24T02:52:27.273 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.26428.log 2026-03-24T02:52:27.273 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.87613.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.87613.log.gz 2026-03-24T02:52:27.273 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64195.log 2026-03-24T02:52:27.275 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.1.26428.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.1.26428.log.gz 2026-03-24T02:52:27.275 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69061.log 2026-03-24T02:52:27.276 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64195.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64195.log.gz 2026-03-24T02:52:27.276 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32478.log 2026-03-24T02:52:27.277 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.69061.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.69061.log.gz 2026-03-24T02:52:27.277 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.87962.log 2026-03-24T02:52:27.278 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32478.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44326.log 2026-03-24T02:52:27.278 INFO:teuthology.orchestra.run.vm07.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32478.log.gz 2026-03-24T02:52:27.280 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.0.87962.log: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.0.87962.log.gz 2026-03-24T02:52:27.280 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39293.log 2026-03-24T02:52:27.280 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.44326.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84128.log 2026-03-24T02:52:27.283 INFO:teuthology.orchestra.run.vm07.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.44326.log.gz 2026-03-24T02:52:27.283 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39293.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39293.log.gz 2026-03-24T02:52:27.284 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78901.log 2026-03-24T02:52:27.284 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84128.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84128.log.gz 2026-03-24T02:52:27.284 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77311.log 2026-03-24T02:52:27.285 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.78901.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.78901.log.gz 2026-03-24T02:52:27.286 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76008.log 2026-03-24T02:52:27.288 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77311.log: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77311.log.gz 2026-03-24T02:52:27.288 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45542.log 2026-03-24T02:52:27.289 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.76008.log: 94.4% -- replaced with /var/log/ceph/cluster1-client.mirror.76008.log.gz 2026-03-24T02:52:27.289 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89655.log 2026-03-24T02:52:27.290 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45542.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45542.log.gz 2026-03-24T02:52:27.290 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35605.log 2026-03-24T02:52:27.290 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.89655.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.89655.log.gz 2026-03-24T02:52:27.290 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86101.log 2026-03-24T02:52:27.291 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35605.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.35605.log.gz 2026-03-24T02:52:27.291 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32008.log 2026-03-24T02:52:27.292 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86101.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.86101.log.gz 2026-03-24T02:52:27.292 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45275.log 2026-03-24T02:52:27.293 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32008.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32008.log.gz 2026-03-24T02:52:27.293 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65112.log 2026-03-24T02:52:27.293 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45275.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76229.log 2026-03-24T02:52:27.306 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65112.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65112.log.gz 2026-03-24T02:52:27.307 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79660.log 2026-03-24T02:52:27.313 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76229.log: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.46353.log.gz 2026-03-24T02:52:27.313 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76229.log.gz 2026-03-24T02:52:27.314 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84196.log 2026-03-24T02:52:27.315 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79660.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.79660.log.gz 2026-03-24T02:52:27.315 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37635.log 2026-03-24T02:52:27.316 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84196.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84196.log.gz 2026-03-24T02:52:27.316 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87355.log 2026-03-24T02:52:27.317 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37635.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37635.log.gz 2026-03-24T02:52:27.317 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84016.log 2026-03-24T02:52:27.318 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.87355.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.87355.log.gz 2026-03-24T02:52:27.318 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34986.log 2026-03-24T02:52:27.318 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84016.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.84016.log.gz 2026-03-24T02:52:27.319 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56459.log 2026-03-24T02:52:27.319 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.34986.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.34986.log.gz 2026-03-24T02:52:27.320 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65023.log 2026-03-24T02:52:27.320 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56459.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56459.log.gz 2026-03-24T02:52:27.320 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44018.log 2026-03-24T02:52:27.321 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65023.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65023.log.gz 2026-03-24T02:52:27.321 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58682.log 2026-03-24T02:52:27.322 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.44018.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85366.log 2026-03-24T02:52:27.325 INFO:teuthology.orchestra.run.vm07.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44018.log.gz 2026-03-24T02:52:27.325 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.58682.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30927.log 2026-03-24T02:52:27.327 INFO:teuthology.orchestra.run.vm07.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58682.log.gz 2026-03-24T02:52:27.327 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85366.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.85366.log.gz 2026-03-24T02:52:27.327 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39760.log 2026-03-24T02:52:27.328 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30927.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.30927.log.gz 2026-03-24T02:52:27.328 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28840.log 2026-03-24T02:52:27.329 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.39760.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.39760.log.gz 2026-03-24T02:52:27.329 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27426.log 2026-03-24T02:52:27.330 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.28840.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.28840.log.gz 2026-03-24T02:52:27.330 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65407.log 2026-03-24T02:52:27.330 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.27426.log: 67.9% -- replaced with /var/log/ceph/cluster2-client.mirror.27426.log.gz 2026-03-24T02:52:27.330 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72595.log 2026-03-24T02:52:27.331 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65407.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59671.log 2026-03-24T02:52:27.331 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65407.log.gz 2026-03-24T02:52:27.334 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72595.log: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.72595.log.gz 2026-03-24T02:52:27.334 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39928.log 2026-03-24T02:52:27.335 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59671.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61640.log 2026-03-24T02:52:27.335 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.59671.log.gz 2026-03-24T02:52:27.335 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39928.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42467.log 2026-03-24T02:52:27.336 INFO:teuthology.orchestra.run.vm07.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39928.log.gz 2026-03-24T02:52:27.336 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.61640.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79297.log 2026-03-24T02:52:27.336 INFO:teuthology.orchestra.run.vm07.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.61640.log.gz 2026-03-24T02:52:27.346 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42467.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81581.log 2026-03-24T02:52:27.347 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.79297.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.79297.log.gz 2026-03-24T02:52:27.347 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45565.log 2026-03-24T02:52:27.355 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81581.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81581.log.gz 2026-03-24T02:52:27.355 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28721.log 2026-03-24T02:52:27.374 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45565.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64230.log 2026-03-24T02:52:27.375 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.28721.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.28721.log.gz 2026-03-24T02:52:27.378 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34063.log 2026-03-24T02:52:27.386 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64230.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.47825.log 2026-03-24T02:52:27.390 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34063.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64230.log.gz 2026-03-24T02:52:27.394 INFO:teuthology.orchestra.run.vm07.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34063.log.gz 2026-03-24T02:52:27.394 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85298.log 2026-03-24T02:52:27.396 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.1.47825.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42467.log.gz 2026-03-24T02:52:27.406 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86026.log 2026-03-24T02:52:27.407 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85298.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.85298.log.gz 2026-03-24T02:52:27.418 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59400.log 2026-03-24T02:52:27.419 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.86026.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.86026.log.gz 2026-03-24T02:52:27.430 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44369.log 2026-03-24T02:52:27.431 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59400.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.59400.log.gz 2026-03-24T02:52:27.442 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37970.log 2026-03-24T02:52:27.445 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.44369.log: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.44369.log.gz 2026-03-24T02:52:27.458 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88339.log 2026-03-24T02:52:27.461 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37970.log: 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.37970.log.gz 2026-03-24T02:52:27.478 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58248.log 2026-03-24T02:52:27.478 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.88339.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88339.log.gz 2026-03-24T02:52:27.490 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2026-03-24T02:52:27.491 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.58248.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58248.log.gz 2026-03-24T02:52:27.502 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43451.log 2026-03-24T02:52:27.510 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55320.log 2026-03-24T02:52:27.518 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43451.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43451.log.gz 2026-03-24T02:52:27.518 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67783.log 2026-03-24T02:52:27.519 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55320.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54459.log 2026-03-24T02:52:27.526 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.67783.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.55320.log.gz 2026-03-24T02:52:27.526 INFO:teuthology.orchestra.run.vm07.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67783.log.gz 2026-03-24T02:52:27.526 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65341.log 2026-03-24T02:52:27.534 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.54459.log: 94.7% -- replaced with /var/log/ceph/cluster1-client.mirror.54459.log.gz 2026-03-24T02:52:27.538 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26319.log 2026-03-24T02:52:27.539 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65341.log: 96.1% 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45565.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.65341.log.gz 2026-03-24T02:52:27.539 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-24T02:52:27.550 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60252.log 2026-03-24T02:52:27.551 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.26319.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.26319.log.gz 2026-03-24T02:52:27.562 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25787.log 2026-03-24T02:52:27.565 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.60252.log: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60252.log.gz 2026-03-24T02:52:27.574 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74058.log 2026-03-24T02:52:27.575 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.25787.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25787.log.gz 2026-03-24T02:52:27.586 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30090.log 2026-03-24T02:52:27.587 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.74058.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74058.log.gz 2026-03-24T02:52:27.598 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84308.log 2026-03-24T02:52:27.599 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30090.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30090.log.gz 2026-03-24T02:52:27.602 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69530.log 2026-03-24T02:52:27.603 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84308.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84308.log.gz 2026-03-24T02:52:27.614 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62210.log 2026-03-24T02:52:27.622 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.69530.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.69530.log.gz 2026-03-24T02:52:27.626 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80354.log 2026-03-24T02:52:27.627 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62210.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62210.log.gz 2026-03-24T02:52:27.627 INFO:teuthology.orchestra.run.vm07.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45275.log.gz 2026-03-24T02:52:27.638 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32101.log 2026-03-24T02:52:27.639 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80354.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.80354.log.gz 2026-03-24T02:52:27.650 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34098.log 2026-03-24T02:52:27.650 INFO:teuthology.orchestra.run.vm07.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-client.mirror.2.39640.log.gz 2026-03-24T02:52:27.651 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32101.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32101.log.gz 2026-03-24T02:52:27.651 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41156.log 2026-03-24T02:52:27.652 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34098.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.34098.log.gz 2026-03-24T02:52:27.652 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41464.log 2026-03-24T02:52:27.652 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41156.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.41156.log.gz 2026-03-24T02:52:27.653 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61685.log 2026-03-24T02:52:27.655 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.41464.log: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.41464.log.gz 2026-03-24T02:52:27.656 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30706.log 2026-03-24T02:52:27.656 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.61685.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49095.log 2026-03-24T02:52:27.657 INFO:teuthology.orchestra.run.vm07.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.61685.log.gz 2026-03-24T02:52:27.657 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30706.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.30706.log.gz 2026-03-24T02:52:27.658 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83633.log 2026-03-24T02:52:27.658 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.49095.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34723.log 2026-03-24T02:52:27.674 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83633.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.83633.log.gz 2026-03-24T02:52:27.674 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70008.log 2026-03-24T02:52:27.675 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34723.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.34723.log.gz 2026-03-24T02:52:27.682 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45428.log 2026-03-24T02:52:27.683 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.70008.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70008.log.gz 2026-03-24T02:52:27.686 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75944.log 2026-03-24T02:52:27.694 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45428.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45428.log.gz 2026-03-24T02:52:27.694 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88750.log 2026-03-24T02:52:27.698 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.75944.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75944.log.gz 2026-03-24T02:52:27.706 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32229.log 2026-03-24T02:52:27.707 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88750.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.88750.log.gz 2026-03-24T02:52:27.707 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43889.log 2026-03-24T02:52:27.710 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32229.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.32229.log.gz 2026-03-24T02:52:27.726 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72287.log 2026-03-24T02:52:27.727 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.43889.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60391.log 2026-03-24T02:52:27.729 INFO:teuthology.orchestra.run.vm07.stderr: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43889.log.gz 2026-03-24T02:52:27.731 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.72287.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.72287.log.gz 2026-03-24T02:52:27.738 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34207.log 2026-03-24T02:52:27.739 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.60391.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.60391.log.gz 2026-03-24T02:52:27.746 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40707.log 2026-03-24T02:52:27.754 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.34207.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43528.log 2026-03-24T02:52:27.754 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.34207.log.gz 2026-03-24T02:52:27.766 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40707.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73633.log 2026-03-24T02:52:27.766 INFO:teuthology.orchestra.run.vm07.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40707.log.gz 2026-03-24T02:52:27.767 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43528.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43528.log.gz 2026-03-24T02:52:27.774 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38035.log 2026-03-24T02:52:27.782 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.73633.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56836.log 2026-03-24T02:52:27.787 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38035.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.73633.log.gz 2026-03-24T02:52:27.790 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33970.log 2026-03-24T02:52:27.798 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56836.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56836.log.gz 2026-03-24T02:52:27.798 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27712.log 2026-03-24T02:52:27.806 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33970.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33970.log.gz 2026-03-24T02:52:27.810 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47533.log 2026-03-24T02:52:27.822 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.27712.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82467.log 2026-03-24T02:52:27.834 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47533.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57247.log 2026-03-24T02:52:27.835 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82467.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.82467.log.gz 2026-03-24T02:52:27.853 INFO:teuthology.orchestra.run.vm07.stderr: 96.0% -- replaced with /var/log/ceph/cluster2-client.mirror.49095.log.gz 2026-03-24T02:52:27.858 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36907.log 2026-03-24T02:52:27.859 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57247.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.57247.log.gz 2026-03-24T02:52:27.870 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56233.log 2026-03-24T02:52:27.871 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36907.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.36907.log.gz 2026-03-24T02:52:27.874 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52794.log 2026-03-24T02:52:27.882 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56233.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56233.log.gz 2026-03-24T02:52:27.882 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71394.log 2026-03-24T02:52:27.883 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.52794.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52794.log.gz 2026-03-24T02:52:27.890 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88690.log 2026-03-24T02:52:27.891 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71394.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71394.log.gz 2026-03-24T02:52:27.891 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30648.log 2026-03-24T02:52:27.891 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88690.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88690.log.gz 2026-03-24T02:52:27.902 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63947.log 2026-03-24T02:52:27.903 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30648.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.30648.log.gz 2026-03-24T02:52:27.903 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45961.log 2026-03-24T02:52:27.910 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.63947.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63947.log.gz 2026-03-24T02:52:27.910 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87100.log 2026-03-24T02:52:27.930 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45961.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61305.log 2026-03-24T02:52:27.930 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87100.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.87100.log.gz 2026-03-24T02:52:27.950 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43869.log 2026-03-24T02:52:27.951 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.61305.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.61305.log.gz 2026-03-24T02:52:27.978 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79773.log 2026-03-24T02:52:27.981 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43869.log: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.43869.log.gz 2026-03-24T02:52:27.995 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79773.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.79773.log.gz 2026-03-24T02:52:27.998 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79987.log 2026-03-24T02:52:28.010 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79895.log 2026-03-24T02:52:28.011 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79987.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.79987.log.gz 2026-03-24T02:52:28.034 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85186.log 2026-03-24T02:52:28.035 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79895.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79895.log.gz 2026-03-24T02:52:28.050 INFO:teuthology.orchestra.run.vm07.stderr: 95.5%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84623.log 2026-03-24T02:52:28.050 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85186.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.85186.log.gz 2026-03-24T02:52:28.052 INFO:teuthology.orchestra.run.vm07.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.38035.log.gz 2026-03-24T02:52:28.052 INFO:teuthology.orchestra.run.vm07.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.45961.log.gz 2026-03-24T02:52:28.054 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63553.log 2026-03-24T02:52:28.059 INFO:teuthology.orchestra.run.vm07.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.27712.log.gz 2026-03-24T02:52:28.066 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84623.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84623.log.gz 2026-03-24T02:52:28.066 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28440.log 2026-03-24T02:52:28.067 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.63553.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63553.log.gz 2026-03-24T02:52:28.070 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47533.log.gz 2026-03-24T02:52:28.070 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76893.log 2026-03-24T02:52:28.071 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.28440.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.28440.log.gz 2026-03-24T02:52:28.071 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33319.log 2026-03-24T02:52:28.071 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76893.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76893.log.gz 2026-03-24T02:52:28.072 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40765.log 2026-03-24T02:52:28.072 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33319.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33319.log.gz 2026-03-24T02:52:28.072 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47234.log 2026-03-24T02:52:28.073 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40765.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40765.log.gz 2026-03-24T02:52:28.073 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59081.log 2026-03-24T02:52:28.074 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47234.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66238.log 2026-03-24T02:52:28.090 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59081.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.59081.log.gz 2026-03-24T02:52:28.091 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69610.log 2026-03-24T02:52:28.092 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.66238.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.66238.log.gz 2026-03-24T02:52:28.102 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84938.log 2026-03-24T02:52:28.103 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.69610.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87060.log 2026-03-24T02:52:28.103 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.69610.log.gz 2026-03-24T02:52:28.103 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84938.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84938.log.gz 2026-03-24T02:52:28.114 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44584.log 2026-03-24T02:52:28.115 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87060.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.87060.log.gz 2026-03-24T02:52:28.126 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38292.log 2026-03-24T02:52:28.127 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.44584.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78747.log 2026-03-24T02:52:28.127 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38292.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86381.log 2026-03-24T02:52:28.127 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78747.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72086.log 2026-03-24T02:52:28.128 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86381.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75657.log 2026-03-24T02:52:28.128 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.78747.log.gz 2026-03-24T02:52:28.130 INFO:teuthology.orchestra.run.vm07.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.44584.log.gz 2026-03-24T02:52:28.130 INFO:teuthology.orchestra.run.vm07.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.38292.log.gz 2026-03-24T02:52:28.140 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72086.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.72086.log.gz 2026-03-24T02:52:28.142 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.86381.log.gz 2026-03-24T02:52:28.154 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62776.log 2026-03-24T02:52:28.155 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.75657.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75657.log.gz 2026-03-24T02:52:28.166 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45252.log 2026-03-24T02:52:28.167 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62776.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.62776.log.gz 2026-03-24T02:52:28.174 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54418.log 2026-03-24T02:52:28.186 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45252.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45252.log.gz 2026-03-24T02:52:28.186 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30276.log 2026-03-24T02:52:28.188 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54418.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.54418.log.gz 2026-03-24T02:52:28.196 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83228.log 2026-03-24T02:52:28.196 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30276.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30276.log.gz 2026-03-24T02:52:28.210 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87528.log 2026-03-24T02:52:28.210 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83228.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47234.log.gz 2026-03-24T02:52:28.210 INFO:teuthology.orchestra.run.vm07.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.83228.log.gz 2026-03-24T02:52:28.211 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88118.log 2026-03-24T02:52:28.222 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87528.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85276.log 2026-03-24T02:52:28.223 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88118.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88118.log.gz 2026-03-24T02:52:28.226 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79365.log 2026-03-24T02:52:28.234 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85276.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.85276.log.gz 2026-03-24T02:52:28.234 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71122.log 2026-03-24T02:52:28.239 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79365.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.79365.log.gz 2026-03-24T02:52:28.239 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46274.log 2026-03-24T02:52:28.246 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71122.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56335.log 2026-03-24T02:52:28.247 INFO:teuthology.orchestra.run.vm07.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.71122.log.gz 2026-03-24T02:52:28.254 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29904.log 2026-03-24T02:52:28.262 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46274.log: /var/log/ceph/cluster2-client.mirror.56335.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56335.log.gz 2026-03-24T02:52:28.262 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73345.log 2026-03-24T02:52:28.270 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29904.log: 88.1%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62865.log 2026-03-24T02:52:28.270 INFO:teuthology.orchestra.run.vm07.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.29904.log.gz 2026-03-24T02:52:28.278 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.73345.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42944.log 2026-03-24T02:52:28.282 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62865.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.73345.log.gz 2026-03-24T02:52:28.282 INFO:teuthology.orchestra.run.vm07.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62865.log.gz 2026-03-24T02:52:28.290 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79394.log 2026-03-24T02:52:28.291 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42944.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.42944.log.gz 2026-03-24T02:52:28.306 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25786.log 2026-03-24T02:52:28.307 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79394.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79394.log.gz 2026-03-24T02:52:28.314 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75172.log 2026-03-24T02:52:28.315 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.25786.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25786.log.gz 2026-03-24T02:52:28.322 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53681.log 2026-03-24T02:52:28.334 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.75172.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65660.log 2026-03-24T02:52:28.334 INFO:teuthology.orchestra.run.vm07.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.75172.log.gz 2026-03-24T02:52:28.335 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.53681.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53681.log.gz 2026-03-24T02:52:28.350 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32380.log 2026-03-24T02:52:28.351 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65660.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65660.log.gz 2026-03-24T02:52:28.358 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78429.log 2026-03-24T02:52:28.366 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32380.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.32380.log.gz 2026-03-24T02:52:28.366 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88445.log 2026-03-24T02:52:28.378 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78429.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.78429.log.gz 2026-03-24T02:52:28.378 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86121.log 2026-03-24T02:52:28.379 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88445.log: 83.4% -- replaced with /var/log/ceph/cluster2-client.mirror.88445.log.gz 2026-03-24T02:52:28.394 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54960.log 2026-03-24T02:52:28.395 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86121.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86121.log.gz 2026-03-24T02:52:28.406 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67338.log 2026-03-24T02:52:28.407 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54960.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54960.log.gz 2026-03-24T02:52:28.418 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68655.log 2026-03-24T02:52:28.419 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.67338.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.67338.log.gz 2026-03-24T02:52:28.430 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82146.log 2026-03-24T02:52:28.431 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68655.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.68655.log.gz 2026-03-24T02:52:28.442 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79345.log 2026-03-24T02:52:28.443 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82146.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82146.log.gz 2026-03-24T02:52:28.450 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33540.log 2026-03-24T02:52:28.451 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79345.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79345.log.gz 2026-03-24T02:52:28.452 INFO:teuthology.orchestra.run.vm07.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46274.log.gz 2026-03-24T02:52:28.452 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39062.log 2026-03-24T02:52:28.453 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33540.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38369.log 2026-03-24T02:52:28.453 INFO:teuthology.orchestra.run.vm07.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33540.log.gz 2026-03-24T02:52:28.454 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39062.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.39062.log.gz 2026-03-24T02:52:28.454 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31578.log 2026-03-24T02:52:28.454 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38369.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86559.log 2026-03-24T02:52:28.455 INFO:teuthology.orchestra.run.vm07.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38369.log.gz 2026-03-24T02:52:28.455 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31578.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31578.log.gz 2026-03-24T02:52:28.456 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29401.log 2026-03-24T02:52:28.456 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.86559.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.86559.log.gz 2026-03-24T02:52:28.456 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44778.log 2026-03-24T02:52:28.457 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29401.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73217.log 2026-03-24T02:52:28.459 INFO:teuthology.orchestra.run.vm07.stderr: 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.29401.log.gz 2026-03-24T02:52:28.460 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.44778.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82864.log 2026-03-24T02:52:28.460 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.44778.log.gz 2026-03-24T02:52:28.460 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.73217.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64550.log 2026-03-24T02:52:28.461 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73217.log.gz 2026-03-24T02:52:28.462 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82864.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82864.log.gz 2026-03-24T02:52:28.462 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43570.log 2026-03-24T02:52:28.463 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.64550.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49548.log 2026-03-24T02:52:28.463 INFO:teuthology.orchestra.run.vm07.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.64550.log.gz 2026-03-24T02:52:28.464 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43570.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.43570.log.gz 2026-03-24T02:52:28.464 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86975.log 2026-03-24T02:52:28.465 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.49548.log: 84.3% -- replaced with /var/log/ceph/cluster2-client.mirror.49548.log.gz 2026-03-24T02:52:28.465 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38257.log 2026-03-24T02:52:28.465 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.86975.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.86975.log.gz 2026-03-24T02:52:28.466 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66409.log 2026-03-24T02:52:28.466 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38257.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38257.log.gz 2026-03-24T02:52:28.466 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64357.log 2026-03-24T02:52:28.467 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66409.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66409.log.gz 2026-03-24T02:52:28.467 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61372.log 2026-03-24T02:52:28.468 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64357.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64357.log.gz 2026-03-24T02:52:28.468 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85762.log 2026-03-24T02:52:28.469 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61372.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85253.log 2026-03-24T02:52:28.473 INFO:teuthology.orchestra.run.vm07.stderr: 97.1% -- replaced with /var/log/ceph/cluster2-client.mirror.61372.log.gz 2026-03-24T02:52:28.473 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85762.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.85762.log.gz 2026-03-24T02:52:28.474 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68949.log 2026-03-24T02:52:28.474 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85253.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.85253.log.gz 2026-03-24T02:52:28.474 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72480.log 2026-03-24T02:52:28.475 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68949.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.68949.log.gz 2026-03-24T02:52:28.475 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74364.log 2026-03-24T02:52:28.476 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.72480.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.72480.log.gz 2026-03-24T02:52:28.476 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68315.log 2026-03-24T02:52:28.476 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.74364.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55588.log 2026-03-24T02:52:28.478 INFO:teuthology.orchestra.run.vm07.stderr: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74364.log.gz 2026-03-24T02:52:28.479 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68315.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.68315.log.gz 2026-03-24T02:52:28.479 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40300.log 2026-03-24T02:52:28.479 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55588.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55588.log.gz 2026-03-24T02:52:28.480 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.26429.log 2026-03-24T02:52:28.480 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40300.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40300.log.gz 2026-03-24T02:52:28.480 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47463.log 2026-03-24T02:52:28.483 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.2.26429.log: 94.1% -- replaced with /var/log/ceph/cluster1-client.mirror.2.26429.log.gz 2026-03-24T02:52:28.483 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55219.log 2026-03-24T02:52:28.484 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47463.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.23828.log 2026-03-24T02:52:28.494 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55219.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55219.log.gz 2026-03-24T02:52:28.494 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34252.log 2026-03-24T02:52:28.495 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.23828.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.23828.log.gz 2026-03-24T02:52:28.496 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87528.log.gz 2026-03-24T02:52:28.502 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55807.log 2026-03-24T02:52:28.512 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.34252.log: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34252.log.gz 2026-03-24T02:52:28.513 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72210.log 2026-03-24T02:52:28.523 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55807.log: 95.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73310.log 2026-03-24T02:52:28.523 INFO:teuthology.orchestra.run.vm07.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.55807.log.gz 2026-03-24T02:52:28.530 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72210.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.72210.log.gz 2026-03-24T02:52:28.530 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61068.log 2026-03-24T02:52:28.531 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.73310.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.73310.log.gz 2026-03-24T02:52:28.531 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46494.log 2026-03-24T02:52:28.538 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61068.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61068.log.gz 2026-03-24T02:52:28.539 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81091.log 2026-03-24T02:52:28.558 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46494.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53995.log 2026-03-24T02:52:28.559 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81091.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81091.log.gz 2026-03-24T02:52:28.566 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25788.log 2026-03-24T02:52:28.574 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53995.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48254.log 2026-03-24T02:52:28.574 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.25788.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25788.log.gz 2026-03-24T02:52:28.578 INFO:teuthology.orchestra.run.vm07.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53995.log.gz 2026-03-24T02:52:28.583 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2026-03-24T02:52:28.594 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.48254.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63979.log 2026-03-24T02:52:28.605 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-osd.0.log: 94.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.27582.log.gz 2026-03-24T02:52:28.605 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70145.log 2026-03-24T02:52:28.605 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.63979.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.63979.log.gz 2026-03-24T02:52:28.606 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58583.log 2026-03-24T02:52:28.606 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.70145.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70145.log.gz 2026-03-24T02:52:28.606 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.log 2026-03-24T02:52:28.607 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.58583.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58583.log.gz 2026-03-24T02:52:28.607 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.74364.log 2026-03-24T02:52:28.610 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2.log: 87.3% -- replaced with /var/log/ceph/cluster2.log.gz 2026-03-24T02:52:28.610 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89447.log 2026-03-24T02:52:28.623 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.3.74364.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.3.74364.log.gz 2026-03-24T02:52:28.624 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35791.log 2026-03-24T02:52:28.624 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.89447.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.89447.log.gz 2026-03-24T02:52:28.624 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28644.log 2026-03-24T02:52:28.624 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35791.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.35791.log.gz 2026-03-24T02:52:28.625 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70591.log 2026-03-24T02:52:28.625 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.28644.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34440.log 2026-03-24T02:52:28.625 INFO:teuthology.orchestra.run.vm07.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.28644.log.gz 2026-03-24T02:52:28.626 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.70591.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70591.log.gz 2026-03-24T02:52:28.626 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37465.log 2026-03-24T02:52:28.627 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.34440.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.34440.log.gz 2026-03-24T02:52:28.627 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29108.log 2026-03-24T02:52:28.627 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37465.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37465.log.gz 2026-03-24T02:52:28.627 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80629.log 2026-03-24T02:52:28.628 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.29108.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.29108.log.gz 2026-03-24T02:52:28.628 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35291.log 2026-03-24T02:52:28.629 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80629.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.80629.log.gz 2026-03-24T02:52:28.629 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32571.log 2026-03-24T02:52:28.629 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35291.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.35291.log.gz 2026-03-24T02:52:28.630 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59205.log 2026-03-24T02:52:28.630 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32571.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32571.log.gz 2026-03-24T02:52:28.631 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56539.log 2026-03-24T02:52:28.631 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59205.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71097.log 2026-03-24T02:52:28.631 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.59205.log.gz 2026-03-24T02:52:28.632 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.56539.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42044.log 2026-03-24T02:52:28.632 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.56539.log.gz 2026-03-24T02:52:28.632 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.71097.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33412.log 2026-03-24T02:52:28.632 INFO:teuthology.orchestra.run.vm07.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.71097.log.gz 2026-03-24T02:52:28.633 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42044.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81346.log 2026-03-24T02:52:28.633 INFO:teuthology.orchestra.run.vm07.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.42044.log.gz 2026-03-24T02:52:28.634 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33412.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33412.log.gz 2026-03-24T02:52:28.634 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54284.log 2026-03-24T02:52:28.635 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81346.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.81346.log.gz 2026-03-24T02:52:28.635 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82722.log 2026-03-24T02:52:28.636 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54284.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54284.log.gz 2026-03-24T02:52:28.636 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84871.log 2026-03-24T02:52:28.637 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82722.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82722.log.gz 2026-03-24T02:52:28.637 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34340.log 2026-03-24T02:52:28.637 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84871.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84871.log.gz 2026-03-24T02:52:28.638 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62394.log 2026-03-24T02:52:28.638 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34340.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78881.log 2026-03-24T02:52:28.650 INFO:teuthology.orchestra.run.vm07.stderr: 95.8%/var/log/ceph/cluster2-client.mirror.62394.log: -- replaced with /var/log/ceph/cluster2-client.mirror.47463.log.gz 2026-03-24T02:52:28.655 INFO:teuthology.orchestra.run.vm07.stderr: 97.2% -- replaced with /var/log/ceph/cluster2-client.mirror.62394.log.gz 2026-03-24T02:52:28.655 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87932.log 2026-03-24T02:52:28.662 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78881.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32761.log 2026-03-24T02:52:28.663 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87932.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.78881.log.gz 2026-03-24T02:52:28.663 INFO:teuthology.orchestra.run.vm07.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.87932.log.gz 2026-03-24T02:52:28.663 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76781.log 2026-03-24T02:52:28.670 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32761.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32761.log.gz 2026-03-24T02:52:28.670 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57417.log 2026-03-24T02:52:28.671 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76781.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76781.log.gz 2026-03-24T02:52:28.678 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72396.log 2026-03-24T02:52:28.683 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57417.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.57417.log.gz 2026-03-24T02:52:28.686 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88200.log 2026-03-24T02:52:28.694 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72396.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72396.log.gz 2026-03-24T02:52:28.694 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36500.log 2026-03-24T02:52:28.695 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88200.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88200.log.gz 2026-03-24T02:52:28.706 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52701.log 2026-03-24T02:52:28.707 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36500.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36500.log.gz 2026-03-24T02:52:28.718 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26118.log 2026-03-24T02:52:28.719 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.52701.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52701.log.gz 2026-03-24T02:52:28.724 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46494.log.gz 2026-03-24T02:52:28.726 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86615.log 2026-03-24T02:52:28.726 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.26118.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.26118.log.gz 2026-03-24T02:52:28.735 INFO:teuthology.orchestra.run.vm07.stderr: 93.8% -- replaced with /var/log/ceph/cluster1-client.mirror.1.47825.log.gz 2026-03-24T02:52:28.746 INFO:teuthology.orchestra.run.vm07.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.34340.log.gz 2026-03-24T02:52:28.746 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84353.log 2026-03-24T02:52:28.747 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86615.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75435.log 2026-03-24T02:52:28.747 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84353.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84353.log.gz 2026-03-24T02:52:28.747 INFO:teuthology.orchestra.run.vm07.stderr: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.86615.log.gz 2026-03-24T02:52:28.750 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57212.log 2026-03-24T02:52:28.751 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.75435.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75435.log.gz 2026-03-24T02:52:28.751 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53569.log 2026-03-24T02:52:28.752 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57212.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.57212.log.gz 2026-03-24T02:52:28.752 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28875.log 2026-03-24T02:52:28.752 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53569.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.53569.log.gz 2026-03-24T02:52:28.753 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63417.log 2026-03-24T02:52:28.753 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.28875.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.28875.log.gz 2026-03-24T02:52:28.753 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35074.log 2026-03-24T02:52:28.754 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.63417.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63417.log.gz 2026-03-24T02:52:28.754 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60727.log 2026-03-24T02:52:28.754 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.35074.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84668.log 2026-03-24T02:52:28.757 INFO:teuthology.orchestra.run.vm07.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.35074.log.gz 2026-03-24T02:52:28.758 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.60727.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69495.log 2026-03-24T02:52:28.758 INFO:teuthology.orchestra.run.vm07.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.60727.log.gz 2026-03-24T02:52:28.759 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84668.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84668.log.gz 2026-03-24T02:52:28.759 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38488.log 2026-03-24T02:52:28.760 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.69495.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.69495.log.gz 2026-03-24T02:52:28.760 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30741.log 2026-03-24T02:52:28.760 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38488.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38488.log.gz 2026-03-24T02:52:28.760 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64046.log 2026-03-24T02:52:28.761 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30741.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30741.log.gz 2026-03-24T02:52:28.761 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46926.log 2026-03-24T02:52:28.762 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64046.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69737.log 2026-03-24T02:52:28.766 INFO:teuthology.orchestra.run.vm07.stderr: 97.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64046.log.gz 2026-03-24T02:52:28.767 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46926.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62260.log 2026-03-24T02:52:28.767 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.69737.log: 84.3% -- replaced with /var/log/ceph/cluster2-client.mirror.46926.log.gz 2026-03-24T02:52:28.767 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69737.log.gz 2026-03-24T02:52:28.767 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71564.log 2026-03-24T02:52:28.768 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62260.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62260.log.gz 2026-03-24T02:52:28.768 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47184.log 2026-03-24T02:52:28.769 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71564.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71564.log.gz 2026-03-24T02:52:28.769 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86998.log 2026-03-24T02:52:28.769 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47184.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28686.log 2026-03-24T02:52:28.782 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70180.log 2026-03-24T02:52:28.782 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.86998.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.86998.log.gz 2026-03-24T02:52:28.783 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.28686.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65224.log 2026-03-24T02:52:28.783 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.70180.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.28686.log.gz 2026-03-24T02:52:28.790 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70180.log.gz 2026-03-24T02:52:28.790 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43200.log 2026-03-24T02:52:28.795 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.65224.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65224.log.gz 2026-03-24T02:52:28.806 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45194.log 2026-03-24T02:52:28.807 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43200.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43200.log.gz 2026-03-24T02:52:28.818 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50445.log 2026-03-24T02:52:28.819 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.45194.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.45194.log.gz 2026-03-24T02:52:28.819 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50927.log 2026-03-24T02:52:28.826 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.50445.log: 2.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50445.log.gz 2026-03-24T02:52:28.826 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28409.log 2026-03-24T02:52:28.830 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.50927.log: 96.1% -- replaced with /var/log/ceph/cluster2-client.mirror.48254.log.gz 2026-03-24T02:52:28.834 INFO:teuthology.orchestra.run.vm07.stderr: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50927.log.gz 2026-03-24T02:52:28.834 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37000.log 2026-03-24T02:52:28.835 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.28409.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.28409.log.gz 2026-03-24T02:52:28.835 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59748.log 2026-03-24T02:52:28.836 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37000.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78712.log 2026-03-24T02:52:28.836 INFO:teuthology.orchestra.run.vm07.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37000.log.gz 2026-03-24T02:52:28.836 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59748.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80397.log 2026-03-24T02:52:28.837 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.59748.log.gz 2026-03-24T02:52:28.837 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78712.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.78712.log.gz 2026-03-24T02:52:28.838 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41198.log 2026-03-24T02:52:28.838 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80397.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.80397.log.gz 2026-03-24T02:52:28.839 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39370.log 2026-03-24T02:52:28.839 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47184.log.gz 2026-03-24T02:52:28.839 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41198.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41198.log.gz 2026-03-24T02:52:28.839 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67068.log 2026-03-24T02:52:28.840 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39370.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47946.log 2026-03-24T02:52:28.840 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.67068.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.39370.log.gz 2026-03-24T02:52:28.840 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.67068.log.gz 2026-03-24T02:52:28.840 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63332.log 2026-03-24T02:52:28.841 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47946.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47946.log.gz 2026-03-24T02:52:28.841 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46978.log 2026-03-24T02:52:28.842 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39729.log 2026-03-24T02:52:28.842 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.63332.log: /var/log/ceph/cluster1-client.mirror.46978.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63332.log.gz 2026-03-24T02:52:28.842 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36163.log 2026-03-24T02:52:28.843 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.39729.log: 94.4% -- replaced with /var/log/ceph/cluster1-client.mirror.46978.log.gz 2026-03-24T02:52:28.843 INFO:teuthology.orchestra.run.vm07.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.admin.39729.log.gz 2026-03-24T02:52:28.843 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36814.log 2026-03-24T02:52:28.843 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28516.log 2026-03-24T02:52:28.844 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36163.log: /var/log/ceph/cluster2-client.mirror.36814.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.36163.log.gz 2026-03-24T02:52:28.844 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 88.4% --verbose -- -- replaced with /var/log/ceph/cluster2-client.mirror.36814.log.gz 2026-03-24T02:52:28.844 INFO:teuthology.orchestra.run.vm07.stderr: /var/log/ceph/cluster2-client.mirror.87638.log 2026-03-24T02:52:28.845 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.28516.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.28516.log.gz 2026-03-24T02:52:28.845 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64133.log 2026-03-24T02:52:28.845 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40672.log 2026-03-24T02:52:28.846 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87638.log: /var/log/ceph/cluster2-client.mirror.64133.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64133.log.gz 2026-03-24T02:52:28.846 INFO:teuthology.orchestra.run.vm07.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.87638.log.gz 2026-03-24T02:52:28.846 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78856.log 2026-03-24T02:52:28.847 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.27290.log 2026-03-24T02:52:28.847 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40672.log: /var/log/ceph/cluster1-client.mirror.78856.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40672.log.gz 2026-03-24T02:52:28.847 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65802.log 2026-03-24T02:52:28.848 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.27290.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.78856.log.gz 2026-03-24T02:52:28.848 INFO:teuthology.orchestra.run.vm07.stderr: 80.3% -- replaced with /var/log/ceph/cluster2-client.admin.27290.log.gz 2026-03-24T02:52:28.848 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79172.log 2026-03-24T02:52:28.849 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65802.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48848.log 2026-03-24T02:52:28.849 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57545.log 2026-03-24T02:52:28.849 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79172.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79172.log.gz 2026-03-24T02:52:28.850 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46471.log 2026-03-24T02:52:28.850 INFO:teuthology.orchestra.run.vm07.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65802.log.gz 2026-03-24T02:52:28.850 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57545.log: /var/log/ceph/cluster2-client.mirror.48848.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.57545.log.gz 2026-03-24T02:52:28.850 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84983.log 2026-03-24T02:52:28.851 INFO:teuthology.orchestra.run.vm07.stderr: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48848.log.gz 2026-03-24T02:52:28.851 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46471.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46471.log.gz 2026-03-24T02:52:28.851 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83186.log 2026-03-24T02:52:28.852 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84983.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56606.log 2026-03-24T02:52:28.852 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83186.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84983.log.gz 2026-03-24T02:52:28.852 INFO:teuthology.orchestra.run.vm07.stderr: 3.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83186.log.gz 2026-03-24T02:52:28.852 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57818.log 2026-03-24T02:52:28.852 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55455.log 2026-03-24T02:52:28.853 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56606.log: /var/log/ceph/cluster2-client.mirror.57818.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.57818.log.gz 2026-03-24T02:52:28.854 INFO:teuthology.orchestra.run.vm07.stderr: 95.2% -- replaced with /var/log/ceph/cluster2-client.mirror.56606.log.gz 2026-03-24T02:52:28.855 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74318.log 2026-03-24T02:52:28.855 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55455.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82345.log 2026-03-24T02:52:28.855 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.74318.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55455.log.gz 2026-03-24T02:52:28.856 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61665.log 2026-03-24T02:52:28.856 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82345.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.82345.log.gz 2026-03-24T02:52:28.857 INFO:teuthology.orchestra.run.vm07.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74318.log.gz 2026-03-24T02:52:28.857 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86183.log 2026-03-24T02:52:28.858 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61665.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87230.log 2026-03-24T02:52:28.858 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86183.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.86183.log.gz 2026-03-24T02:52:28.859 INFO:teuthology.orchestra.run.vm07.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61665.log.gz 2026-03-24T02:52:28.859 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29641.log 2026-03-24T02:52:28.859 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87230.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44147.log 2026-03-24T02:52:28.860 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29641.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78967.log 2026-03-24T02:52:28.873 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.44147.log: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.44147.log.gz 2026-03-24T02:52:28.874 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83081.log 2026-03-24T02:52:28.882 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78967.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.78967.log.gz 2026-03-24T02:52:28.882 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68060.log 2026-03-24T02:52:28.891 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.83081.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83081.log.gz 2026-03-24T02:52:28.891 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76318.log 2026-03-24T02:52:28.892 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68060.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87399.log 2026-03-24T02:52:28.892 INFO:teuthology.orchestra.run.vm07.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.68060.log.gz 2026-03-24T02:52:28.898 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76318.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82631.log 2026-03-24T02:52:28.898 INFO:teuthology.orchestra.run.vm07.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76318.log.gz 2026-03-24T02:52:28.899 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87399.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83723.log 2026-03-24T02:52:28.899 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.87399.log.gz 2026-03-24T02:52:28.906 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82631.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.82631.log.gz 2026-03-24T02:52:28.906 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48231.log 2026-03-24T02:52:28.914 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83723.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.87230.log.gz 2026-03-24T02:52:28.914 INFO:teuthology.orchestra.run.vm07.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.83723.log.gz 2026-03-24T02:52:28.914 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80510.log 2026-03-24T02:52:28.915 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.48231.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48231.log.gz 2026-03-24T02:52:28.918 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66847.log 2026-03-24T02:52:28.920 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80510.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80510.log.gz 2026-03-24T02:52:28.922 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57352.log 2026-03-24T02:52:28.923 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66847.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66847.log.gz 2026-03-24T02:52:28.926 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42086.log 2026-03-24T02:52:28.927 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57352.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.57352.log.gz 2026-03-24T02:52:28.930 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85388.log 2026-03-24T02:52:28.931 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42086.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.42086.log.gz 2026-03-24T02:52:28.931 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64392.log 2026-03-24T02:52:28.932 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85388.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.85388.log.gz 2026-03-24T02:52:28.932 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86462.log 2026-03-24T02:52:28.932 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64392.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64392.log.gz 2026-03-24T02:52:28.933 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52736.log 2026-03-24T02:52:28.933 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86462.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.86462.log.gz 2026-03-24T02:52:28.934 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40521.log 2026-03-24T02:52:28.934 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.52736.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52736.log.gz 2026-03-24T02:52:28.934 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31729.log 2026-03-24T02:52:28.935 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40521.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40521.log.gz 2026-03-24T02:52:28.935 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67415.log 2026-03-24T02:52:28.936 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31729.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31729.log.gz 2026-03-24T02:52:28.936 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88648.log 2026-03-24T02:52:28.936 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.67415.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85118.log 2026-03-24T02:52:28.937 INFO:teuthology.orchestra.run.vm07.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67415.log.gz 2026-03-24T02:52:28.938 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.88648.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.88648.log.gz 2026-03-24T02:52:28.938 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75707.log 2026-03-24T02:52:28.939 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85118.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.85118.log.gz 2026-03-24T02:52:28.939 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37870.log 2026-03-24T02:52:28.939 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.75707.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75707.log.gz 2026-03-24T02:52:28.940 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29309.log 2026-03-24T02:52:28.940 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37870.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.37870.log.gz 2026-03-24T02:52:28.940 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81817.log 2026-03-24T02:52:28.941 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29309.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.29309.log.gz 2026-03-24T02:52:28.941 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72744.log 2026-03-24T02:52:28.942 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81817.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.81817.log.gz 2026-03-24T02:52:28.942 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82509.log 2026-03-24T02:52:28.943 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72744.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72744.log.gz 2026-03-24T02:52:28.943 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62014.log 2026-03-24T02:52:28.944 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82509.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82509.log.gz 2026-03-24T02:52:28.945 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41233.log 2026-03-24T02:52:28.945 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62014.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68630.log 2026-03-24T02:52:28.950 INFO:teuthology.orchestra.run.vm07.stderr: 97.1% -- replaced with /var/log/ceph/cluster2-client.mirror.62014.log.gz 2026-03-24T02:52:28.950 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41233.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65246.log 2026-03-24T02:52:28.950 INFO:teuthology.orchestra.run.vm07.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.41233.log.gz 2026-03-24T02:52:28.951 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.68630.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32947.log 2026-03-24T02:52:28.951 INFO:teuthology.orchestra.run.vm07.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.68630.log.gz 2026-03-24T02:52:28.952 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65246.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83746.log 2026-03-24T02:52:28.957 INFO:teuthology.orchestra.run.vm07.stderr: 97.2% -- replaced with /var/log/ceph/cluster2-client.mirror.65246.log.gz 2026-03-24T02:52:28.957 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32947.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32947.log.gz 2026-03-24T02:52:28.957 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53048.log 2026-03-24T02:52:28.958 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83746.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.83746.log.gz 2026-03-24T02:52:28.958 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76122.log 2026-03-24T02:52:28.959 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53048.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53048.log.gz 2026-03-24T02:52:28.959 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85456.log 2026-03-24T02:52:28.959 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76122.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76122.log.gz 2026-03-24T02:52:28.959 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.46108.log 2026-03-24T02:52:28.960 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85456.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.85456.log.gz 2026-03-24T02:52:28.960 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40579.log 2026-03-24T02:52:28.960 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.0.46108.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80439.log 2026-03-24T02:52:28.971 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40579.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40579.log.gz 2026-03-24T02:52:28.982 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26792.log 2026-03-24T02:52:28.983 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80439.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80439.log.gz 2026-03-24T02:52:28.984 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65948.log 2026-03-24T02:52:28.990 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.26792.log: 72.3% -- replaced with /var/log/ceph/cluster1-client.admin.26792.log.gz 2026-03-24T02:52:28.990 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87733.log 2026-03-24T02:52:28.996 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65948.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.0.46108.log.gz 2026-03-24T02:52:28.997 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65948.log.gz 2026-03-24T02:52:28.997 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85556.log 2026-03-24T02:52:28.997 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87733.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.87733.log.gz 2026-03-24T02:52:28.998 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47777.log 2026-03-24T02:52:28.998 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.85556.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.85556.log.gz 2026-03-24T02:52:28.998 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65822.log 2026-03-24T02:52:28.999 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47777.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47777.log.gz 2026-03-24T02:52:28.999 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44607.log 2026-03-24T02:52:28.999 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.65822.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68003.log 2026-03-24T02:52:29.000 INFO:teuthology.orchestra.run.vm07.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.65822.log.gz 2026-03-24T02:52:29.001 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44607.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44607.log.gz 2026-03-24T02:52:29.001 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83768.log 2026-03-24T02:52:29.002 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68003.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.68003.log.gz 2026-03-24T02:52:29.002 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86302.log 2026-03-24T02:52:29.002 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83768.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83768.log.gz 2026-03-24T02:52:29.003 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88512.log 2026-03-24T02:52:29.008 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86302.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60192.log 2026-03-24T02:52:29.020 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88512.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75742.log 2026-03-24T02:52:29.031 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.60192.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.60192.log.gz 2026-03-24T02:52:29.046 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44435.log 2026-03-24T02:52:29.047 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.75742.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75742.log.gz 2026-03-24T02:52:29.055 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42979.log 2026-03-24T02:52:29.061 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44435.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44435.log.gz 2026-03-24T02:52:29.066 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.85812.log 2026-03-24T02:52:29.066 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42979.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.42979.log.gz 2026-03-24T02:52:29.071 INFO:teuthology.orchestra.run.vm07.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.29641.log.gz 2026-03-24T02:52:29.078 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66312.log 2026-03-24T02:52:29.081 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.1.85812.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.1.85812.log.gz 2026-03-24T02:52:29.090 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39104.log 2026-03-24T02:52:29.091 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66312.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.66312.log.gz 2026-03-24T02:52:29.093 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.88512.log.gz 2026-03-24T02:52:29.094 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59860.log 2026-03-24T02:52:29.094 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39104.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.39104.log.gz 2026-03-24T02:52:29.094 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42340.log 2026-03-24T02:52:29.095 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59860.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59860.log.gz 2026-03-24T02:52:29.095 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38555.log 2026-03-24T02:52:29.096 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42340.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.26437.log 2026-03-24T02:52:29.096 INFO:teuthology.orchestra.run.vm07.stderr: 95.3% -- replaced with /var/log/ceph/cluster2-client.mirror.86302.log.gz 2026-03-24T02:52:29.099 INFO:teuthology.orchestra.run.vm07.stderr: 96.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42340.log.gz 2026-03-24T02:52:29.099 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.38555.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.38555.log.gz 2026-03-24T02:52:29.099 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67586.log 2026-03-24T02:52:29.100 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.3.26437.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.26485.log 2026-03-24T02:52:29.101 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.67586.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.67586.log.gz 2026-03-24T02:52:29.110 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31264.log 2026-03-24T02:52:29.113 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.1.26485.log: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.1.26485.log.gz 2026-03-24T02:52:29.126 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86484.log 2026-03-24T02:52:29.127 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31264.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.31264.log.gz 2026-03-24T02:52:29.138 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42790.log 2026-03-24T02:52:29.139 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86484.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86484.log.gz 2026-03-24T02:52:29.146 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82217.log 2026-03-24T02:52:29.146 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42790.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.42790.log.gz 2026-03-24T02:52:29.154 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.23058.log 2026-03-24T02:52:29.155 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82217.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82217.log.gz 2026-03-24T02:52:29.156 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82560.log 2026-03-24T02:52:29.156 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.23058.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.23058.log.gz 2026-03-24T02:52:29.156 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85433.log 2026-03-24T02:52:29.157 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82560.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82560.log.gz 2026-03-24T02:52:29.157 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82680.log 2026-03-24T02:52:29.157 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85433.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.85433.log.gz 2026-03-24T02:52:29.157 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75334.log 2026-03-24T02:52:29.158 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82680.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.82680.log.gz 2026-03-24T02:52:29.159 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86059.log 2026-03-24T02:52:29.159 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.75334.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75334.log.gz 2026-03-24T02:52:29.159 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50146.log 2026-03-24T02:52:29.160 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.86059.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.86059.log.gz 2026-03-24T02:52:29.160 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26344.log 2026-03-24T02:52:29.161 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.50146.log: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50146.log.gz 2026-03-24T02:52:29.161 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84376.log 2026-03-24T02:52:29.161 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.26344.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.26344.log.gz 2026-03-24T02:52:29.161 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27341.log 2026-03-24T02:52:29.162 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84376.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84376.log.gz 2026-03-24T02:52:29.162 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85321.log 2026-03-24T02:52:29.163 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.27341.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.27341.log.gz 2026-03-24T02:52:29.163 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87040.log 2026-03-24T02:52:29.163 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85321.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.85321.log.gz 2026-03-24T02:52:29.164 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78016.log 2026-03-24T02:52:29.164 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87040.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.87040.log.gz 2026-03-24T02:52:29.165 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26144.log 2026-03-24T02:52:29.166 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78016.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.78016.log.gz 2026-03-24T02:52:29.166 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43998.log 2026-03-24T02:52:29.166 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.26144.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.26144.log.gz 2026-03-24T02:52:29.167 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32287.log 2026-03-24T02:52:29.167 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43998.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77924.log 2026-03-24T02:52:29.170 INFO:teuthology.orchestra.run.vm07.stderr: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43998.log.gz 2026-03-24T02:52:29.171 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32287.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63046.log 2026-03-24T02:52:29.171 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77924.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32287.log.gz 2026-03-24T02:52:29.171 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77924.log.gz 2026-03-24T02:52:29.171 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36070.log 2026-03-24T02:52:29.173 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.63046.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63046.log.gz 2026-03-24T02:52:29.173 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45167.log 2026-03-24T02:52:29.173 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36070.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.36070.log.gz 2026-03-24T02:52:29.174 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80949.log 2026-03-24T02:52:29.174 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.45167.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.45167.log.gz 2026-03-24T02:52:29.175 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83858.log 2026-03-24T02:52:29.175 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80949.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80949.log.gz 2026-03-24T02:52:29.175 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62529.log 2026-03-24T02:52:29.176 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83858.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.83858.log.gz 2026-03-24T02:52:29.176 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.46978.log 2026-03-24T02:52:29.177 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62529.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62529.log.gz 2026-03-24T02:52:29.177 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86142.log 2026-03-24T02:52:29.186 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.1.46978.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54604.log 2026-03-24T02:52:29.188 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86142.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.1.46978.log.gz 2026-03-24T02:52:29.188 INFO:teuthology.orchestra.run.vm07.stderr: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.86142.log.gz 2026-03-24T02:52:29.189 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59027.log 2026-03-24T02:52:29.189 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54604.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54604.log.gz 2026-03-24T02:52:29.189 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78058.log 2026-03-24T02:52:29.190 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59027.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59027.log.gz 2026-03-24T02:52:29.190 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83296.log 2026-03-24T02:52:29.191 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78058.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79943.log 2026-03-24T02:52:29.194 INFO:teuthology.orchestra.run.vm07.stderr: 96.5% -- replaced with /var/log/ceph/cluster2-client.mirror.78058.log.gz 2026-03-24T02:52:29.194 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83296.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81723.log 2026-03-24T02:52:29.194 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79943.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83296.log.gz 2026-03-24T02:52:29.195 INFO:teuthology.orchestra.run.vm07.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79943.log.gz 2026-03-24T02:52:29.195 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53703.log 2026-03-24T02:52:29.196 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81723.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39811.log 2026-03-24T02:52:29.197 INFO:teuthology.orchestra.run.vm07.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.81723.log.gz 2026-03-24T02:52:29.202 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53703.log: 97.1% -- replaced with /var/log/ceph/cluster2-client.mirror.53703.log.gz 2026-03-24T02:52:29.202 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48706.log 2026-03-24T02:52:29.203 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.39811.log: 85.4% -- replaced with /var/log/ceph/cluster1-client.mirror.39811.log.gz 2026-03-24T02:52:29.203 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88486.log 2026-03-24T02:52:29.203 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.48706.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48706.log.gz 2026-03-24T02:52:29.203 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88592.log 2026-03-24T02:52:29.204 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88486.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88486.log.gz 2026-03-24T02:52:29.204 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30427.log 2026-03-24T02:52:29.214 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88592.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44908.log 2026-03-24T02:52:29.215 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30427.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30427.log.gz 2026-03-24T02:52:29.219 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79702.log 2026-03-24T02:52:29.222 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44908.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44908.log.gz 2026-03-24T02:52:29.226 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66525.log 2026-03-24T02:52:29.230 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79702.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.88592.log.gz 2026-03-24T02:52:29.231 INFO:teuthology.orchestra.run.vm07.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.79702.log.gz 2026-03-24T02:52:29.232 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33261.log 2026-03-24T02:52:29.232 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66525.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66525.log.gz 2026-03-24T02:52:29.232 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60615.log 2026-03-24T02:52:29.233 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33261.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.33261.log.gz 2026-03-24T02:52:29.233 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74853.log 2026-03-24T02:52:29.233 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.60615.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35096.log 2026-03-24T02:52:29.234 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.60615.log.gz 2026-03-24T02:52:29.234 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.74853.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.74853.log.gz 2026-03-24T02:52:29.235 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81439.log 2026-03-24T02:52:29.235 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35096.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66347.log 2026-03-24T02:52:29.240 INFO:teuthology.orchestra.run.vm07.stderr: 97.0% -- replaced with /var/log/ceph/cluster2-client.mirror.35096.log.gz 2026-03-24T02:52:29.240 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81439.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43912.log 2026-03-24T02:52:29.241 INFO:teuthology.orchestra.run.vm07.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81439.log.gz 2026-03-24T02:52:29.242 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66347.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66347.log.gz 2026-03-24T02:52:29.242 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66983.log 2026-03-24T02:52:29.242 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43912.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85787.log 2026-03-24T02:52:29.245 INFO:teuthology.orchestra.run.vm07.stderr: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.43912.log.gz 2026-03-24T02:52:29.245 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66983.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66983.log.gz 2026-03-24T02:52:29.246 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32982.log 2026-03-24T02:52:29.247 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.85787.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85787.log.gz 2026-03-24T02:52:29.247 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60230.log 2026-03-24T02:52:29.247 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32982.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.32982.log.gz 2026-03-24T02:52:29.247 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33784.log 2026-03-24T02:52:29.248 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.60230.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.60230.log.gz 2026-03-24T02:52:29.249 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87880.log 2026-03-24T02:52:29.249 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33784.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33784.log.gz 2026-03-24T02:52:29.249 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72906.log 2026-03-24T02:52:29.250 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.87880.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.87880.log.gz 2026-03-24T02:52:29.250 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56055.log 2026-03-24T02:52:29.251 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72906.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72906.log.gz 2026-03-24T02:52:29.251 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.68720.log 2026-03-24T02:52:29.252 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56055.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56055.log.gz 2026-03-24T02:52:29.252 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39027.log 2026-03-24T02:52:29.253 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.0.68720.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71487.log 2026-03-24T02:52:29.263 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39027.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39027.log.gz 2026-03-24T02:52:29.266 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30834.log 2026-03-24T02:52:29.270 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71487.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.71487.log.gz 2026-03-24T02:52:29.274 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58019.log 2026-03-24T02:52:29.278 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30834.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.30834.log.gz 2026-03-24T02:52:29.281 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72267.log 2026-03-24T02:52:29.282 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.58019.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.58019.log.gz 2026-03-24T02:52:29.290 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59825.log 2026-03-24T02:52:29.291 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72267.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.72267.log.gz 2026-03-24T02:52:29.298 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81859.log 2026-03-24T02:52:29.299 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59825.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.59825.log.gz 2026-03-24T02:52:29.302 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53362.log 2026-03-24T02:52:29.310 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81859.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81859.log.gz 2026-03-24T02:52:29.311 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55077.log 2026-03-24T02:52:29.313 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53362.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53362.log.gz 2026-03-24T02:52:29.322 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46712.log 2026-03-24T02:52:29.323 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.55077.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55077.log.gz 2026-03-24T02:52:29.338 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77507.log 2026-03-24T02:52:29.338 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46712.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49917.log 2026-03-24T02:52:29.346 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77507.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48821.log 2026-03-24T02:52:29.350 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.49917.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77507.log.gz 2026-03-24T02:52:29.355 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82394.log 2026-03-24T02:52:29.366 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.48821.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26394.log 2026-03-24T02:52:29.367 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82394.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.82394.log.gz 2026-03-24T02:52:29.378 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77371.log 2026-03-24T02:52:29.378 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.26394.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.26394.log.gz 2026-03-24T02:52:29.400 INFO:teuthology.orchestra.run.vm07.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.68720.log.gz 2026-03-24T02:52:29.402 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85141.log 2026-03-24T02:52:29.403 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77371.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77371.log.gz 2026-03-24T02:52:29.410 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30462.log 2026-03-24T02:52:29.418 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85141.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.85141.log.gz 2026-03-24T02:52:29.426 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82893.log 2026-03-24T02:52:29.427 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30462.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30462.log.gz 2026-03-24T02:52:29.430 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78270.log 2026-03-24T02:52:29.433 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82893.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.82893.log.gz 2026-03-24T02:52:29.435 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61915.log 2026-03-24T02:52:29.439 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.78270.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.78270.log.gz 2026-03-24T02:52:29.446 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86209.log 2026-03-24T02:52:29.454 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61915.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.61915.log.gz 2026-03-24T02:52:29.462 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87502.log 2026-03-24T02:52:29.474 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86209.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30520.log 2026-03-24T02:52:29.475 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87502.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.87502.log.gz 2026-03-24T02:52:29.486 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58640.log 2026-03-24T02:52:29.487 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30520.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30520.log.gz 2026-03-24T02:52:29.502 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58154.log 2026-03-24T02:52:29.503 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.58640.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30555.log 2026-03-24T02:52:29.504 INFO:teuthology.orchestra.run.vm07.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58640.log.gz 2026-03-24T02:52:29.504 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.58154.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46712.log.gz 2026-03-24T02:52:29.504 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.58154.log.gz 2026-03-24T02:52:29.522 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49443.log 2026-03-24T02:52:29.523 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30555.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30555.log.gz 2026-03-24T02:52:29.526 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.22965.log 2026-03-24T02:52:29.534 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.49443.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49443.log.gz 2026-03-24T02:52:29.534 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85028.log 2026-03-24T02:52:29.535 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.22965.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.22965.log.gz 2026-03-24T02:52:29.539 INFO:teuthology.orchestra.run.vm07.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.49917.log.gz 2026-03-24T02:52:29.539 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26775.log 2026-03-24T02:52:29.540 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85028.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.85028.log.gz 2026-03-24T02:52:29.548 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39479.log 2026-03-24T02:52:29.550 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.26775.log: 80.3% -- replaced with /var/log/ceph/cluster1-client.admin.26775.log.gz 2026-03-24T02:52:29.554 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56584.log 2026-03-24T02:52:29.556 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.39479.log: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.39479.log.gz 2026-03-24T02:52:29.570 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81902.log 2026-03-24T02:52:29.571 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.56584.log: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56584.log.gz 2026-03-24T02:52:29.577 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47969.log 2026-03-24T02:52:29.586 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81902.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.81902.log.gz 2026-03-24T02:52:29.587 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30334.log 2026-03-24T02:52:29.598 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47969.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70486.log 2026-03-24T02:52:29.599 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30334.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.30334.log.gz 2026-03-24T02:52:29.600 INFO:teuthology.orchestra.run.vm07.stderr: 96.0% -- replaced with /var/log/ceph/cluster2-client.mirror.48821.log.gz 2026-03-24T02:52:29.614 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37558.log 2026-03-24T02:52:29.614 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.70486.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70486.log.gz 2026-03-24T02:52:29.616 INFO:teuthology.orchestra.run.vm07.stderr: 96.1% -- replaced with /var/log/ceph/cluster2-client.mirror.86209.log.gz 2026-03-24T02:52:29.626 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78789.log 2026-03-24T02:52:29.627 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37558.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.37558.log.gz 2026-03-24T02:52:29.638 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83926.log 2026-03-24T02:52:29.639 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78789.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.78789.log.gz 2026-03-24T02:52:29.639 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80221.log 2026-03-24T02:52:29.646 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83926.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83926.log.gz 2026-03-24T02:52:29.646 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73074.log 2026-03-24T02:52:29.655 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80221.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80221.log.gz 2026-03-24T02:52:29.655 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40393.log 2026-03-24T02:52:29.656 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.73074.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73074.log.gz 2026-03-24T02:52:29.666 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35756.log 2026-03-24T02:52:29.667 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40393.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40393.log.gz 2026-03-24T02:52:29.667 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86360.log 2026-03-24T02:52:29.674 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35756.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.35756.log.gz 2026-03-24T02:52:29.674 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55708.log 2026-03-24T02:52:29.682 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.86360.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.86360.log.gz 2026-03-24T02:52:29.694 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86893.log 2026-03-24T02:52:29.695 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55708.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.55708.log.gz 2026-03-24T02:52:29.714 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60488.log 2026-03-24T02:52:29.715 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.86893.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.86893.log.gz 2026-03-24T02:52:29.722 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46755.log 2026-03-24T02:52:29.730 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.60488.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47331.log 2026-03-24T02:52:29.734 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46755.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60488.log.gz 2026-03-24T02:52:29.734 INFO:teuthology.orchestra.run.vm07.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46755.log.gz 2026-03-24T02:52:29.738 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33633.log 2026-03-24T02:52:29.746 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47331.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30055.log 2026-03-24T02:52:29.754 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33633.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61573.log 2026-03-24T02:52:29.758 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30055.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33633.log.gz 2026-03-24T02:52:29.758 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.30055.log.gz 2026-03-24T02:52:29.766 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43826.log 2026-03-24T02:52:29.767 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61573.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61573.log.gz 2026-03-24T02:52:29.774 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72779.log 2026-03-24T02:52:29.784 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43826.log: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.43826.log.gz 2026-03-24T02:52:29.785 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87907.log 2026-03-24T02:52:29.785 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72779.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.72779.log.gz 2026-03-24T02:52:29.794 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86402.log 2026-03-24T02:52:29.794 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.87907.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.87907.log.gz 2026-03-24T02:52:29.806 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41774.log 2026-03-24T02:52:29.807 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86402.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.86402.log.gz 2026-03-24T02:52:29.814 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38580.log 2026-03-24T02:52:29.818 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41774.log: 96.1% -- replaced with /var/log/ceph/cluster2-client.mirror.47969.log.gz 2026-03-24T02:52:29.819 INFO:teuthology.orchestra.run.vm07.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41774.log.gz 2026-03-24T02:52:29.830 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77959.log 2026-03-24T02:52:29.831 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38580.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82104.log 2026-03-24T02:52:29.833 INFO:teuthology.orchestra.run.vm07.stderr: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.38580.log.gz 2026-03-24T02:52:29.838 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77959.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74558.log 2026-03-24T02:52:29.842 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82104.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77959.log.gz 2026-03-24T02:52:29.843 INFO:teuthology.orchestra.run.vm07.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.82104.log.gz 2026-03-24T02:52:29.854 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84691.log 2026-03-24T02:52:29.855 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.74558.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74558.log.gz 2026-03-24T02:52:29.867 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84511.log 2026-03-24T02:52:29.867 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84691.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84691.log.gz 2026-03-24T02:52:29.879 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.49354.log 2026-03-24T02:52:29.879 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84511.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84511.log.gz 2026-03-24T02:52:29.890 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68563.log 2026-03-24T02:52:29.900 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.2.49354.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.2.49354.log.gz 2026-03-24T02:52:29.910 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59311.log 2026-03-24T02:52:29.911 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68563.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.68563.log.gz 2026-03-24T02:52:29.922 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26014.log 2026-03-24T02:52:29.923 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59311.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.59311.log.gz 2026-03-24T02:52:29.930 INFO:teuthology.orchestra.run.vm07.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.47331.log.gz 2026-03-24T02:52:29.930 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82990.log 2026-03-24T02:52:29.930 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.26014.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69772.log 2026-03-24T02:52:29.931 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37754.log 2026-03-24T02:52:29.931 INFO:teuthology.orchestra.run.vm07.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.26014.log.gz 2026-03-24T02:52:29.932 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82990.log: /var/log/ceph/cluster2-client.mirror.69772.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.82990.log.gz 2026-03-24T02:52:29.932 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69772.log.gz 2026-03-24T02:52:29.932 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53911.log 2026-03-24T02:52:29.932 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37754.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40986.log 2026-03-24T02:52:29.933 INFO:teuthology.orchestra.run.vm07.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37754.log.gz 2026-03-24T02:52:29.933 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53911.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53911.log.gz 2026-03-24T02:52:29.933 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66745.log 2026-03-24T02:52:29.934 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40986.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40986.log.gz 2026-03-24T02:52:29.934 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73499.log 2026-03-24T02:52:29.935 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66745.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47825.log 2026-03-24T02:52:29.936 INFO:teuthology.orchestra.run.vm07.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.66745.log.gz 2026-03-24T02:52:29.936 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.73499.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26945.log 2026-03-24T02:52:29.936 INFO:teuthology.orchestra.run.vm07.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.73499.log.gz 2026-03-24T02:52:29.937 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47825.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88567.log 2026-03-24T02:52:29.946 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.26945.log: 54.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26945.log.gz 2026-03-24T02:52:29.954 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59346.log 2026-03-24T02:52:29.955 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.88567.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.88567.log.gz 2026-03-24T02:52:29.955 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29490.log 2026-03-24T02:52:29.962 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59346.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59346.log.gz 2026-03-24T02:52:29.962 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70910.log 2026-03-24T02:52:29.967 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29490.log: 97.0% -- replaced with /var/log/ceph/cluster2-client.mirror.29490.log.gz 2026-03-24T02:52:29.977 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71599.log 2026-03-24T02:52:29.978 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.70910.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42198.log 2026-03-24T02:52:29.978 INFO:teuthology.orchestra.run.vm07.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.70910.log.gz 2026-03-24T02:52:29.979 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71599.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71599.log.gz 2026-03-24T02:52:29.990 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41352.log 2026-03-24T02:52:29.991 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42198.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35663.log 2026-03-24T02:52:29.991 INFO:teuthology.orchestra.run.vm07.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42198.log.gz 2026-03-24T02:52:29.991 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41352.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41352.log.gz 2026-03-24T02:52:30.002 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57131.log 2026-03-24T02:52:30.003 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35663.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.35663.log.gz 2026-03-24T02:52:30.003 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57394.log 2026-03-24T02:52:30.003 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57131.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.57131.log.gz 2026-03-24T02:52:30.014 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84466.log 2026-03-24T02:52:30.015 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57394.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55405.log 2026-03-24T02:52:30.022 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.52157.log 2026-03-24T02:52:30.030 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55405.log: /var/log/ceph/cluster1-client.mirror.84466.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84466.log.gz 2026-03-24T02:52:30.030 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55405.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77870.log 2026-03-24T02:52:30.030 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-24T02:52:30.038 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.0.52157.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.88908.log 2026-03-24T02:52:30.039 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77870.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77870.log.gz 2026-03-24T02:52:30.062 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63588.log 2026-03-24T02:52:30.063 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.1.88908.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.1.88908.log.gz 2026-03-24T02:52:30.074 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52613.log 2026-03-24T02:52:30.075 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.63588.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63588.log.gz 2026-03-24T02:52:30.086 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34954.log 2026-03-24T02:52:30.087 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.52613.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52613.log.gz 2026-03-24T02:52:30.098 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43079.log 2026-03-24T02:52:30.099 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34954.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.34954.log.gz 2026-03-24T02:52:30.110 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58660.log 2026-03-24T02:52:30.113 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43079.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43079.log.gz 2026-03-24T02:52:30.126 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63088.log 2026-03-24T02:52:30.127 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.58660.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.58660.log.gz 2026-03-24T02:52:30.132 INFO:teuthology.orchestra.run.vm07.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57394.log.gz 2026-03-24T02:52:30.132 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87440.log 2026-03-24T02:52:30.140 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.63088.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50167.log 2026-03-24T02:52:30.141 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87440.log: 96.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63088.log.gz 2026-03-24T02:52:30.142 INFO:teuthology.orchestra.run.vm07.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.87440.log.gz 2026-03-24T02:52:30.158 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34175.log 2026-03-24T02:52:30.159 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.50167.log: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50167.log.gz 2026-03-24T02:52:30.170 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54755.log 2026-03-24T02:52:30.171 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34175.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34175.log.gz 2026-03-24T02:52:30.182 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35698.log 2026-03-24T02:52:30.183 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54755.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54755.log.gz 2026-03-24T02:52:30.194 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84601.log 2026-03-24T02:52:30.195 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35698.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.35698.log.gz 2026-03-24T02:52:30.206 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54925.log 2026-03-24T02:52:30.207 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84601.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84601.log.gz 2026-03-24T02:52:30.218 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63645.log 2026-03-24T02:52:30.219 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54925.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54925.log.gz 2026-03-24T02:52:30.230 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71429.log 2026-03-24T02:52:30.231 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.63645.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63645.log.gz 2026-03-24T02:52:30.242 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39181.log 2026-03-24T02:52:30.243 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71429.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71429.log.gz 2026-03-24T02:52:30.250 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56755.log 2026-03-24T02:52:30.258 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39181.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.39181.log.gz 2026-03-24T02:52:30.258 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85343.log 2026-03-24T02:52:30.270 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56755.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56755.log.gz 2026-03-24T02:52:30.270 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48679.log 2026-03-24T02:52:30.271 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85343.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.85343.log.gz 2026-03-24T02:52:30.278 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36872.log 2026-03-24T02:52:30.294 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.48679.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55285.log 2026-03-24T02:52:30.295 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36872.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36872.log.gz 2026-03-24T02:52:30.299 INFO:teuthology.orchestra.run.vm07.stderr: 94.0% -- replaced with /var/log/ceph/cluster1-client.mirror.0.52157.log.gz 2026-03-24T02:52:30.312 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46201.log 2026-03-24T02:52:30.312 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55285.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55285.log.gz 2026-03-24T02:52:30.326 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71676.log 2026-03-24T02:52:30.327 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46201.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46201.log.gz 2026-03-24T02:52:30.342 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.89306.log 2026-03-24T02:52:30.343 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71676.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71676.log.gz 2026-03-24T02:52:30.348 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67702.log 2026-03-24T02:52:30.348 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.89306.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.89306.log.gz 2026-03-24T02:52:30.358 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39589.log 2026-03-24T02:52:30.359 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.67702.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.67702.log.gz 2026-03-24T02:52:30.365 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71216.log 2026-03-24T02:52:30.378 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39589.log: 96.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88306.log 2026-03-24T02:52:30.378 INFO:teuthology.orchestra.run.vm07.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.39589.log.gz 2026-03-24T02:52:30.379 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71216.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71216.log.gz 2026-03-24T02:52:30.394 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67437.log 2026-03-24T02:52:30.406 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88306.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73957.log 2026-03-24T02:52:30.412 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.67437.log: 97.2% -- replaced with /var/log/ceph/cluster2-client.mirror.67437.log.gz 2026-03-24T02:52:30.422 INFO:teuthology.orchestra.run.vm07.stderr: 95.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44992.log 2026-03-24T02:52:30.423 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.73957.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73957.log.gz 2026-03-24T02:52:30.434 INFO:teuthology.orchestra.run.vm07.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.48679.log.gz 2026-03-24T02:52:30.434 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68102.log 2026-03-24T02:52:30.435 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44992.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44992.log.gz 2026-03-24T02:52:30.446 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61472.log 2026-03-24T02:52:30.456 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68102.log: 96.8% -- replaced with /var/log/ceph/cluster2-client.mirror.68102.log.gz 2026-03-24T02:52:30.457 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72309.log 2026-03-24T02:52:30.457 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61472.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.61472.log.gz 2026-03-24T02:52:30.466 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38088.log 2026-03-24T02:52:30.478 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72309.log: 97.0% -- replaced with /var/log/ceph/cluster2-client.mirror.72309.log.gz 2026-03-24T02:52:30.478 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68720.log 2026-03-24T02:52:30.479 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.38088.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.38088.log.gz 2026-03-24T02:52:30.482 INFO:teuthology.orchestra.run.vm07.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88306.log.gz 2026-03-24T02:52:30.494 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29029.log 2026-03-24T02:52:30.506 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68720.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66787.log 2026-03-24T02:52:30.509 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.29029.log: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.29029.log.gz 2026-03-24T02:52:30.526 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83521.log 2026-03-24T02:52:30.531 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66787.log: 97.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66787.log.gz 2026-03-24T02:52:30.532 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44628.log 2026-03-24T02:52:30.542 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83521.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37430.log 2026-03-24T02:52:30.542 INFO:teuthology.orchestra.run.vm07.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83521.log.gz 2026-03-24T02:52:30.543 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44628.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44628.log.gz 2026-03-24T02:52:30.554 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62494.log 2026-03-24T02:52:30.555 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37430.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37430.log.gz 2026-03-24T02:52:30.560 INFO:teuthology.orchestra.run.vm07.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.68720.log.gz 2026-03-24T02:52:30.566 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29376.log 2026-03-24T02:52:30.566 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62494.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62494.log.gz 2026-03-24T02:52:30.590 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79487.log 2026-03-24T02:52:30.591 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.29376.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.29376.log.gz 2026-03-24T02:52:30.591 INFO:teuthology.orchestra.run.vm07.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47825.log.gz 2026-03-24T02:52:30.591 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43099.log 2026-03-24T02:52:30.592 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79487.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41121.log 2026-03-24T02:52:30.592 INFO:teuthology.orchestra.run.vm07.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79487.log.gz 2026-03-24T02:52:30.593 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.43099.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66720.log 2026-03-24T02:52:30.595 INFO:teuthology.orchestra.run.vm07.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.43099.log.gz 2026-03-24T02:52:30.596 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41121.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.41121.log.gz 2026-03-24T02:52:30.596 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56144.log 2026-03-24T02:52:30.596 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.66720.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.66720.log.gz 2026-03-24T02:52:30.597 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54396.log 2026-03-24T02:52:30.597 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56144.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45890.log 2026-03-24T02:52:30.597 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56144.log.gz 2026-03-24T02:52:30.598 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.54396.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53217.log 2026-03-24T02:52:30.599 INFO:teuthology.orchestra.run.vm07.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.54396.log.gz 2026-03-24T02:52:30.610 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45890.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41387.log 2026-03-24T02:52:30.612 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53217.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53217.log.gz 2026-03-24T02:52:30.622 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74670.log 2026-03-24T02:52:30.623 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41387.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81304.log 2026-03-24T02:52:30.623 INFO:teuthology.orchestra.run.vm07.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41387.log.gz 2026-03-24T02:52:30.631 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.74670.log: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.74670.log.gz 2026-03-24T02:52:30.631 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36628.log 2026-03-24T02:52:30.632 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81304.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81304.log.gz 2026-03-24T02:52:30.642 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44754.log 2026-03-24T02:52:30.643 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36628.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30148.log 2026-03-24T02:52:30.643 INFO:teuthology.orchestra.run.vm07.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.36628.log.gz 2026-03-24T02:52:30.650 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44754.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44754.log.gz 2026-03-24T02:52:30.662 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52634.log 2026-03-24T02:52:30.663 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30148.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.30148.log.gz 2026-03-24T02:52:30.663 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72860.log 2026-03-24T02:52:30.670 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.52634.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52634.log.gz 2026-03-24T02:52:30.670 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32513.log 2026-03-24T02:52:30.678 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72860.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72860.log.gz 2026-03-24T02:52:30.678 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54790.log 2026-03-24T02:52:30.686 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32513.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32513.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64793.log 2026-03-24T02:52:30.686 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-24T02:52:30.690 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54790.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54790.log.gz 2026-03-24T02:52:30.702 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63620.log 2026-03-24T02:52:30.703 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64793.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64793.log.gz 2026-03-24T02:52:30.714 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62954.log 2026-03-24T02:52:30.715 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.63620.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.63620.log.gz 2026-03-24T02:52:30.722 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36349.log 2026-03-24T02:52:30.730 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62954.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26219.log 2026-03-24T02:52:30.730 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36349.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.62954.log.gz 2026-03-24T02:52:30.738 INFO:teuthology.orchestra.run.vm07.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.36349.log.gz 2026-03-24T02:52:30.738 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48798.log 2026-03-24T02:52:30.738 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.26219.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26219.log.gz 2026-03-24T02:52:30.750 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67370.log 2026-03-24T02:52:30.751 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.48798.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48798.log.gz 2026-03-24T02:52:30.758 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44104.log 2026-03-24T02:52:30.770 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.67370.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80651.log 2026-03-24T02:52:30.770 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.67370.log.gz 2026-03-24T02:52:30.773 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.44104.log: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.44104.log.gz 2026-03-24T02:52:30.778 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45337.log 2026-03-24T02:52:30.786 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80651.log: 96.1%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39447.log 2026-03-24T02:52:30.786 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45337.log: -- replaced with /var/log/ceph/cluster2-client.mirror.45890.log.gz 2026-03-24T02:52:30.786 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80651.log.gz 2026-03-24T02:52:30.798 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43374.log 2026-03-24T02:52:30.799 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39447.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39447.log.gz 2026-03-24T02:52:30.802 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68883.log 2026-03-24T02:52:30.810 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43374.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.43374.log.gz 2026-03-24T02:52:30.810 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75909.log 2026-03-24T02:52:30.811 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68883.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.68883.log.gz 2026-03-24T02:52:30.822 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32194.log 2026-03-24T02:52:30.823 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.75909.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27409.log 2026-03-24T02:52:30.823 INFO:teuthology.orchestra.run.vm07.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75909.log.gz 2026-03-24T02:52:30.830 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32194.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32194.log.gz 2026-03-24T02:52:30.830 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69151.log 2026-03-24T02:52:30.838 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.27409.log: 64.8% -- replaced with /var/log/ceph/cluster2-client.mirror.27409.log.gz 2026-03-24T02:52:30.838 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47114.log 2026-03-24T02:52:30.839 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.69151.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.69151.log.gz 2026-03-24T02:52:30.850 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49022.log 2026-03-24T02:52:30.851 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47114.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37789.log 2026-03-24T02:52:30.858 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.49022.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41851.log 2026-03-24T02:52:30.859 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37789.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49022.log.gz 2026-03-24T02:52:30.866 INFO:teuthology.orchestra.run.vm07.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.37789.log.gz 2026-03-24T02:52:30.866 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46646.log 2026-03-24T02:52:30.874 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41851.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79436.log 2026-03-24T02:52:30.878 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46646.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.41851.log.gz 2026-03-24T02:52:30.886 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74593.log 2026-03-24T02:52:30.888 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79436.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.79436.log.gz 2026-03-24T02:52:30.898 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41681.log 2026-03-24T02:52:30.899 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.74593.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.74593.log.gz 2026-03-24T02:52:30.906 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81879.log 2026-03-24T02:52:30.914 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41681.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.41681.log.gz 2026-03-24T02:52:30.918 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35119.log 2026-03-24T02:52:30.920 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81879.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81879.log.gz 2026-03-24T02:52:30.926 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44041.log 2026-03-24T02:52:30.938 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35119.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88281.log 2026-03-24T02:52:30.941 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44041.log: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.44041.log.gz 2026-03-24T02:52:30.942 INFO:teuthology.orchestra.run.vm07.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45337.log.gz 2026-03-24T02:52:30.942 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35384.log 2026-03-24T02:52:30.943 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.88281.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.88281.log.gz 2026-03-24T02:52:30.954 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51001.log 2026-03-24T02:52:30.955 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35384.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.35384.log.gz 2026-03-24T02:52:30.978 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32854.log 2026-03-24T02:52:30.990 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.51001.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64098.log 2026-03-24T02:52:30.991 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32854.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.32854.log.gz 2026-03-24T02:52:30.998 INFO:teuthology.orchestra.run.vm07.stderr: 94.2% -- replaced with /var/log/ceph/cluster1-client.mirror.3.26437.log.gz 2026-03-24T02:52:30.999 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.28984.log 2026-03-24T02:52:30.999 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64098.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.64098.log.gz 2026-03-24T02:52:30.999 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42363.log 2026-03-24T02:52:31.000 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.28984.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.28984.log.gz 2026-03-24T02:52:31.000 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55099.log 2026-03-24T02:52:31.001 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42363.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44392.log 2026-03-24T02:52:31.015 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55099.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82489.log 2026-03-24T02:52:31.030 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44392.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69635.log 2026-03-24T02:52:31.031 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82489.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82489.log.gz 2026-03-24T02:52:31.033 INFO:teuthology.orchestra.run.vm07.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44392.log.gz 2026-03-24T02:52:31.038 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84758.log 2026-03-24T02:52:31.047 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.69635.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.69635.log.gz 2026-03-24T02:52:31.047 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29139.log 2026-03-24T02:52:31.054 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84758.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84758.log.gz 2026-03-24T02:52:31.058 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54670.log 2026-03-24T02:52:31.059 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.29139.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.29139.log.gz 2026-03-24T02:52:31.061 INFO:teuthology.orchestra.run.vm07.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55099.log.gz 2026-03-24T02:52:31.062 INFO:teuthology.orchestra.run.vm07.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.46646.log.gz 2026-03-24T02:52:31.088 INFO:teuthology.orchestra.run.vm07.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47114.log.gz 2026-03-24T02:52:31.088 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71065.log 2026-03-24T02:52:31.098 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54670.log: 88.0%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44190.log 2026-03-24T02:52:31.098 INFO:teuthology.orchestra.run.vm07.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.54670.log.gz 2026-03-24T02:52:31.099 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71065.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71065.log.gz 2026-03-24T02:52:31.114 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79465.log 2026-03-24T02:52:31.117 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.44190.log: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.44190.log.gz 2026-03-24T02:52:31.125 INFO:teuthology.orchestra.run.vm07.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42363.log.gz 2026-03-24T02:52:31.125 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89022.log 2026-03-24T02:52:31.126 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79465.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89084.log 2026-03-24T02:52:31.126 INFO:teuthology.orchestra.run.vm07.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79465.log.gz 2026-03-24T02:52:31.127 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.89022.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89022.log.gz 2026-03-24T02:52:31.127 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45059.log 2026-03-24T02:52:31.128 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.89084.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89084.log.gz 2026-03-24T02:52:31.128 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67395.log 2026-03-24T02:52:31.129 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45059.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.45059.log.gz 2026-03-24T02:52:31.129 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35326.log 2026-03-24T02:52:31.130 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.67395.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29811.log 2026-03-24T02:52:31.131 INFO:teuthology.orchestra.run.vm07.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.67395.log.gz 2026-03-24T02:52:31.131 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35326.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64688.log 2026-03-24T02:52:31.132 INFO:teuthology.orchestra.run.vm07.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35326.log.gz 2026-03-24T02:52:31.132 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29811.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.29811.log.gz 2026-03-24T02:52:31.132 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25865.log 2026-03-24T02:52:31.133 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64688.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31671.log 2026-03-24T02:52:31.133 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64688.log.gz 2026-03-24T02:52:31.133 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.25865.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25865.log.gz 2026-03-24T02:52:31.134 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45670.log 2026-03-24T02:52:31.134 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31671.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31671.log.gz 2026-03-24T02:52:31.134 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59950.log 2026-03-24T02:52:31.135 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45670.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38411.log 2026-03-24T02:52:31.144 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59950.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55184.log 2026-03-24T02:52:31.145 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38411.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59950.log.gz 2026-03-24T02:52:31.146 INFO:teuthology.orchestra.run.vm07.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.38411.log.gz 2026-03-24T02:52:31.150 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44061.log 2026-03-24T02:52:31.158 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55184.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55184.log.gz 2026-03-24T02:52:31.158 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82416.log 2026-03-24T02:52:31.159 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.44061.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86443.log 2026-03-24T02:52:31.162 INFO:teuthology.orchestra.run.vm07.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44061.log.gz 2026-03-24T02:52:31.170 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82416.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82416.log.gz 2026-03-24T02:52:31.170 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63382.log 2026-03-24T02:52:31.171 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86443.log: 83.5% -- replaced with /var/log/ceph/cluster2-client.mirror.86443.log.gz 2026-03-24T02:52:31.171 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59116.log 2026-03-24T02:52:31.178 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.63382.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63382.log.gz 2026-03-24T02:52:31.178 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48136.log 2026-03-24T02:52:31.179 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59116.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.59116.log.gz 2026-03-24T02:52:31.179 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.38688.log 2026-03-24T02:52:31.194 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.48136.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28551.log 2026-03-24T02:52:31.206 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.2.38688.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72121.log 2026-03-24T02:52:31.207 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.28551.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.28551.log.gz 2026-03-24T02:52:31.213 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84083.log 2026-03-24T02:52:31.214 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72121.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72121.log.gz 2026-03-24T02:52:31.221 INFO:teuthology.orchestra.run.vm07.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.2.38688.log.gz 2026-03-24T02:52:31.221 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70875.log 2026-03-24T02:52:31.222 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84083.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84083.log.gz 2026-03-24T02:52:31.230 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89421.log 2026-03-24T02:52:31.231 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.70875.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70875.log.gz 2026-03-24T02:52:31.242 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81774.log 2026-03-24T02:52:31.243 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.89421.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.89421.log.gz 2026-03-24T02:52:31.254 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71775.log 2026-03-24T02:52:31.255 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81774.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81774.log.gz 2026-03-24T02:52:31.266 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83110.log 2026-03-24T02:52:31.270 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71775.log: 96.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71775.log.gz 2026-03-24T02:52:31.278 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76816.log 2026-03-24T02:52:31.279 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.83110.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.83110.log.gz 2026-03-24T02:52:31.290 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83431.log 2026-03-24T02:52:31.291 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76816.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76816.log.gz 2026-03-24T02:52:31.298 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29009.log 2026-03-24T02:52:31.302 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83431.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83431.log.gz 2026-03-24T02:52:31.307 INFO:teuthology.orchestra.run.vm07.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51001.log.gz 2026-03-24T02:52:31.308 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30799.log 2026-03-24T02:52:31.308 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29009.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62074.log 2026-03-24T02:52:31.311 INFO:teuthology.orchestra.run.vm07.stderr: 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.29009.log.gz 2026-03-24T02:52:31.311 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.30799.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30799.log.gz 2026-03-24T02:52:31.311 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39612.log 2026-03-24T02:52:31.312 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62074.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62074.log.gz 2026-03-24T02:52:31.312 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63021.log 2026-03-24T02:52:31.312 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39612.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40893.log 2026-03-24T02:52:31.326 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.63021.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.63021.log.gz 2026-03-24T02:52:31.326 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33226.log 2026-03-24T02:52:31.327 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40893.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40893.log.gz 2026-03-24T02:52:31.330 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45123.log 2026-03-24T02:52:31.334 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33226.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33226.log.gz 2026-03-24T02:52:31.334 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41886.log 2026-03-24T02:52:31.338 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45123.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.45123.log.gz 2026-03-24T02:52:31.346 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88404.log 2026-03-24T02:52:31.347 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41886.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65179.log 2026-03-24T02:52:31.347 INFO:teuthology.orchestra.run.vm07.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41886.log.gz 2026-03-24T02:52:31.347 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88404.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.88404.log.gz 2026-03-24T02:52:31.362 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71708.log 2026-03-24T02:52:31.363 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.65179.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.65179.log.gz 2026-03-24T02:52:31.370 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76497.log 2026-03-24T02:52:31.371 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.71708.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.71708.log.gz 2026-03-24T02:52:31.386 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41044.log 2026-03-24T02:52:31.387 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.76497.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.76497.log.gz 2026-03-24T02:52:31.388 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48136.log.gz 2026-03-24T02:52:31.394 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83678.log 2026-03-24T02:52:31.395 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41044.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41044.log.gz 2026-03-24T02:52:31.400 INFO:teuthology.orchestra.run.vm07.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.45670.log.gz 2026-03-24T02:52:31.400 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54198.log 2026-03-24T02:52:31.400 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83678.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83678.log.gz 2026-03-24T02:52:31.401 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43011.log 2026-03-24T02:52:31.401 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54198.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35184.log 2026-03-24T02:52:31.402 INFO:teuthology.orchestra.run.vm07.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54198.log.gz 2026-03-24T02:52:31.402 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.43011.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77472.log 2026-03-24T02:52:31.403 INFO:teuthology.orchestra.run.vm07.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43011.log.gz 2026-03-24T02:52:31.403 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.35184.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60027.log 2026-03-24T02:52:31.403 INFO:teuthology.orchestra.run.vm07.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.admin.35184.log.gz 2026-03-24T02:52:31.404 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77472.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77472.log.gz 2026-03-24T02:52:31.404 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74650.log 2026-03-24T02:52:31.405 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.60027.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.60027.log.gz 2026-03-24T02:52:31.405 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83207.log 2026-03-24T02:52:31.405 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.74650.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.42489.log 2026-03-24T02:52:31.406 INFO:teuthology.orchestra.run.vm07.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74650.log.gz 2026-03-24T02:52:31.407 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83207.log: 2.6% -- replaced with /var/log/ceph/cluster1-client.mirror.83207.log.gz 2026-03-24T02:52:31.407 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83498.log 2026-03-24T02:52:31.418 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.1.42489.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37835.log 2026-03-24T02:52:31.419 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83498.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83498.log.gz 2026-03-24T02:52:31.419 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33819.log 2026-03-24T02:52:31.422 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37835.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37835.log.gz 2026-03-24T02:52:31.434 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80058.log 2026-03-24T02:52:31.435 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33819.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33819.log.gz 2026-03-24T02:52:31.435 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43165.log 2026-03-24T02:52:31.435 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80058.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29962.log 2026-03-24T02:52:31.436 INFO:teuthology.orchestra.run.vm07.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80058.log.gz 2026-03-24T02:52:31.443 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43165.log: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.39612.log.gz 2026-03-24T02:52:31.446 INFO:teuthology.orchestra.run.vm07.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.43165.log.gz 2026-03-24T02:52:31.446 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87019.log 2026-03-24T02:52:31.447 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29962.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31392.log 2026-03-24T02:52:31.447 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87019.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.87019.log.gz 2026-03-24T02:52:31.450 INFO:teuthology.orchestra.run.vm07.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29962.log.gz 2026-03-24T02:52:31.458 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89237.log 2026-03-24T02:52:31.459 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31392.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31392.log.gz 2026-03-24T02:52:31.459 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25866.log 2026-03-24T02:52:31.459 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.89237.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.89237.log.gz 2026-03-24T02:52:31.460 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39258.log 2026-03-24T02:52:31.460 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.25866.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25866.log.gz 2026-03-24T02:52:31.461 INFO:teuthology.orchestra.run.vm07.stderr: 96.1% -- replaced with /var/log/ceph/cluster2-client.mirror.35119.log.gz 2026-03-24T02:52:31.461 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40114.log 2026-03-24T02:52:31.462 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39258.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39258.log.gz 2026-03-24T02:52:31.470 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77047.log 2026-03-24T02:52:31.471 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40114.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40114.log.gz 2026-03-24T02:52:31.471 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33877.log 2026-03-24T02:52:31.471 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.77047.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39567.log 2026-03-24T02:52:31.472 INFO:teuthology.orchestra.run.vm07.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.77047.log.gz 2026-03-24T02:52:31.473 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33877.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33877.log.gz 2026-03-24T02:52:31.473 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82126.log 2026-03-24T02:52:31.473 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.39567.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61947.log 2026-03-24T02:52:31.476 INFO:teuthology.orchestra.run.vm07.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.39567.log.gz 2026-03-24T02:52:31.476 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82126.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74625.log 2026-03-24T02:52:31.477 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.82126.log.gz 2026-03-24T02:52:31.477 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.61947.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84556.log 2026-03-24T02:52:31.477 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.61947.log.gz 2026-03-24T02:52:31.478 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.74625.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.74625.log.gz 2026-03-24T02:52:31.478 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80855.log 2026-03-24T02:52:31.478 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84556.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84556.log.gz 2026-03-24T02:52:31.479 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79682.log 2026-03-24T02:52:31.479 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80855.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52157.log 2026-03-24T02:52:31.480 INFO:teuthology.orchestra.run.vm07.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80855.log.gz 2026-03-24T02:52:31.480 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79682.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27582.log 2026-03-24T02:52:31.481 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79682.log.gz 2026-03-24T02:52:31.482 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.52157.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52157.log.gz 2026-03-24T02:52:31.482 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58104.log 2026-03-24T02:52:31.490 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.27582.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41809.log 2026-03-24T02:52:31.491 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.58104.log: 94.5% -- replaced with /var/log/ceph/cluster1-client.mirror.1.42489.log.gz 2026-03-24T02:52:31.491 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87586.log 2026-03-24T02:52:31.492 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41809.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41809.log.gz 2026-03-24T02:52:31.492 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34005.log 2026-03-24T02:52:31.492 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.87586.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.87586.log.gz 2026-03-24T02:52:31.493 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69016.log 2026-03-24T02:52:31.493 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34005.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34005.log.gz 2026-03-24T02:52:31.493 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65575.log 2026-03-24T02:52:31.494 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.69016.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32889.log 2026-03-24T02:52:31.494 INFO:teuthology.orchestra.run.vm07.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58104.log.gz 2026-03-24T02:52:31.494 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.69016.log.gz 2026-03-24T02:52:31.495 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65575.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45367.log 2026-03-24T02:52:31.495 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65575.log.gz 2026-03-24T02:52:31.495 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32889.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.32889.log.gz 2026-03-24T02:52:31.496 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43339.log 2026-03-24T02:52:31.496 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45367.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45367.log.gz 2026-03-24T02:52:31.496 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70260.log 2026-03-24T02:52:31.497 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43339.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.43339.log.gz 2026-03-24T02:52:31.497 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32322.log 2026-03-24T02:52:31.497 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.70260.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.70260.log.gz 2026-03-24T02:52:31.498 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77027.log 2026-03-24T02:52:31.498 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32322.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32322.log.gz 2026-03-24T02:52:31.498 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83152.log 2026-03-24T02:52:31.499 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77027.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44234.log 2026-03-24T02:52:31.500 INFO:teuthology.orchestra.run.vm07.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77027.log.gz 2026-03-24T02:52:31.500 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.83152.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79137.log 2026-03-24T02:52:31.501 INFO:teuthology.orchestra.run.vm07.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.83152.log.gz 2026-03-24T02:52:31.504 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.44234.log: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.44234.log.gz 2026-03-24T02:52:31.504 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86590.log 2026-03-24T02:52:31.505 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79137.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79137.log.gz 2026-03-24T02:52:31.505 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29344.log 2026-03-24T02:52:31.505 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.86590.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.86590.log.gz 2026-03-24T02:52:31.505 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27443.log 2026-03-24T02:52:31.506 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29344.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29344.log.gz 2026-03-24T02:52:31.506 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85231.log 2026-03-24T02:52:31.507 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.27443.log: 56.0% -- replaced with /var/log/ceph/cluster2-client.mirror.27443.log.gz 2026-03-24T02:52:31.507 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84488.log 2026-03-24T02:52:31.507 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85231.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81133.log 2026-03-24T02:52:31.507 INFO:teuthology.orchestra.run.vm07.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.85231.log.gz 2026-03-24T02:52:31.508 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84488.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84488.log.gz 2026-03-24T02:52:31.508 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57984.log 2026-03-24T02:52:31.509 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81133.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47672.log 2026-03-24T02:52:31.509 INFO:teuthology.orchestra.run.vm07.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81133.log.gz 2026-03-24T02:52:31.510 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57984.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.57984.log.gz 2026-03-24T02:52:31.510 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44971.log 2026-03-24T02:52:31.511 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47672.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50015.log 2026-03-24T02:52:31.523 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44971.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44971.log.gz 2026-03-24T02:52:31.523 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44521.log 2026-03-24T02:52:31.538 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.50015.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29193.log 2026-03-24T02:52:31.542 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44521.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88159.log 2026-03-24T02:52:31.545 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29193.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44521.log.gz 2026-03-24T02:52:31.546 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89629.log 2026-03-24T02:52:31.546 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88159.log: 83.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88159.log.gz 2026-03-24T02:52:31.550 INFO:teuthology.orchestra.run.vm07.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.29193.log.gz 2026-03-24T02:52:31.566 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78923.log 2026-03-24T02:52:31.566 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.89629.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.89629.log.gz 2026-03-24T02:52:31.578 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62327.log 2026-03-24T02:52:31.583 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78923.log: 97.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78923.log.gz 2026-03-24T02:52:31.594 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53806.log 2026-03-24T02:52:31.594 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.62327.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.62327.log.gz 2026-03-24T02:52:31.606 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85411.log 2026-03-24T02:52:31.607 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53806.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53806.log.gz 2026-03-24T02:52:31.618 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67524.log 2026-03-24T02:52:31.619 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85411.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.85411.log.gz 2026-03-24T02:52:31.630 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43783.log 2026-03-24T02:52:31.631 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.67524.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.67524.log.gz 2026-03-24T02:52:31.642 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54875.log 2026-03-24T02:52:31.645 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43783.log: 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.43783.log.gz 2026-03-24T02:52:31.658 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40858.log 2026-03-24T02:52:31.659 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54875.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54875.log.gz 2026-03-24T02:52:31.670 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.50242.log 2026-03-24T02:52:31.671 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40858.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40858.log.gz 2026-03-24T02:52:31.678 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29997.log 2026-03-24T02:52:31.686 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.1.50242.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89282.log 2026-03-24T02:52:31.691 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29997.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47672.log.gz 2026-03-24T02:52:31.691 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87205.log 2026-03-24T02:52:31.692 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.89282.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.89282.log.gz 2026-03-24T02:52:31.692 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80468.log 2026-03-24T02:52:31.692 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.87205.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.87205.log.gz 2026-03-24T02:52:31.693 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69923.log 2026-03-24T02:52:31.693 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80468.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34842.log 2026-03-24T02:52:31.693 INFO:teuthology.orchestra.run.vm07.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.80468.log.gz 2026-03-24T02:52:31.694 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.69923.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29997.log.gz 2026-03-24T02:52:31.694 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69923.log.gz 2026-03-24T02:52:31.694 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62639.log 2026-03-24T02:52:31.695 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34842.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.34842.log.gz 2026-03-24T02:52:31.695 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62295.log 2026-03-24T02:52:31.695 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62639.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62639.log.gz 2026-03-24T02:52:31.696 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80600.log 2026-03-24T02:52:31.696 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62295.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62295.log.gz 2026-03-24T02:52:31.696 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41310.log 2026-03-24T02:52:31.697 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80600.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85006.log 2026-03-24T02:52:31.698 INFO:teuthology.orchestra.run.vm07.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.80600.log.gz 2026-03-24T02:52:31.698 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41310.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84826.log 2026-03-24T02:52:31.699 INFO:teuthology.orchestra.run.vm07.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.41310.log.gz 2026-03-24T02:52:31.699 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85006.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.85006.log.gz 2026-03-24T02:52:31.699 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56564.log 2026-03-24T02:52:31.700 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84826.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84826.log.gz 2026-03-24T02:52:31.700 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39963.log 2026-03-24T02:52:31.701 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56564.log: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.56564.log.gz 2026-03-24T02:52:31.701 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88999.log 2026-03-24T02:52:31.702 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39963.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68517.log 2026-03-24T02:52:31.702 INFO:teuthology.orchestra.run.vm07.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39963.log.gz 2026-03-24T02:52:31.702 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.88999.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88999.log.gz 2026-03-24T02:52:31.703 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27375.log 2026-03-24T02:52:31.703 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68517.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83881.log 2026-03-24T02:52:31.703 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.68517.log.gz 2026-03-24T02:52:31.704 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.27375.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.27375.log.gz 2026-03-24T02:52:31.704 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41419.log 2026-03-24T02:52:31.704 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83881.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.83881.log.gz 2026-03-24T02:52:31.705 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71997.log 2026-03-24T02:52:31.705 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.41419.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.41419.log.gz 2026-03-24T02:52:31.705 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82580.log 2026-03-24T02:52:31.706 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71997.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71997.log.gz 2026-03-24T02:52:31.706 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56658.log 2026-03-24T02:52:31.707 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82580.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56020.log 2026-03-24T02:52:31.707 INFO:teuthology.orchestra.run.vm07.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82580.log.gz 2026-03-24T02:52:31.708 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56658.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56658.log.gz 2026-03-24T02:52:31.708 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37677.log 2026-03-24T02:52:31.709 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56020.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56020.log.gz 2026-03-24T02:52:31.709 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59435.log 2026-03-24T02:52:31.710 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37677.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.37677.log.gz 2026-03-24T02:52:31.710 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80835.log 2026-03-24T02:52:31.710 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59435.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68598.log 2026-03-24T02:52:31.711 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.59435.log.gz 2026-03-24T02:52:31.711 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80835.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.80835.log.gz 2026-03-24T02:52:31.711 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31020.log 2026-03-24T02:52:31.712 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68598.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68598.log.gz 2026-03-24T02:52:31.712 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62989.log 2026-03-24T02:52:31.713 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31020.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31020.log.gz 2026-03-24T02:52:31.713 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33598.log 2026-03-24T02:52:31.713 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.62989.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62989.log.gz 2026-03-24T02:52:31.713 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46580.log 2026-03-24T02:52:31.714 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33598.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33598.log.gz 2026-03-24T02:52:31.714 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80813.log 2026-03-24T02:52:31.721 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46580.log: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.1.50242.log.gz 2026-03-24T02:52:31.726 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45101.log 2026-03-24T02:52:31.727 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80813.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.80813.log.gz 2026-03-24T02:52:31.727 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84173.log 2026-03-24T02:52:31.734 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45101.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.45101.log.gz 2026-03-24T02:52:31.735 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43493.log 2026-03-24T02:52:31.742 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84173.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84173.log.gz 2026-03-24T02:52:31.742 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86770.log 2026-03-24T02:52:31.743 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43493.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.43493.log.gz 2026-03-24T02:52:31.743 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73464.log 2026-03-24T02:52:31.750 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86770.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.86770.log.gz 2026-03-24T02:52:31.750 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57731.log 2026-03-24T02:52:31.758 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.73464.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73464.log.gz 2026-03-24T02:52:31.758 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84713.log 2026-03-24T02:52:31.759 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57731.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.57731.log.gz 2026-03-24T02:52:31.759 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56370.log 2026-03-24T02:52:31.766 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84713.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84713.log.gz 2026-03-24T02:52:31.767 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25964.log 2026-03-24T02:52:31.774 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56370.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77177.log 2026-03-24T02:52:31.774 INFO:teuthology.orchestra.run.vm07.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56370.log.gz 2026-03-24T02:52:31.774 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.25964.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25964.log.gz 2026-03-24T02:52:31.784 INFO:teuthology.orchestra.run.vm07.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.46580.log.gz 2026-03-24T02:52:31.784 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67865.log 2026-03-24T02:52:31.784 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77177.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63842.log 2026-03-24T02:52:31.785 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77177.log.gz 2026-03-24T02:52:31.785 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.67865.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53125.log 2026-03-24T02:52:31.786 INFO:teuthology.orchestra.run.vm07.stderr: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67865.log.gz 2026-03-24T02:52:31.786 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.63842.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63842.log.gz 2026-03-24T02:52:31.787 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69838.log 2026-03-24T02:52:31.787 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53125.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53125.log.gz 2026-03-24T02:52:31.787 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89679.log 2026-03-24T02:52:31.788 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.69838.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69838.log.gz 2026-03-24T02:52:31.788 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66080.log 2026-03-24T02:52:31.789 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.89679.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.89679.log.gz 2026-03-24T02:52:31.789 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82294.log 2026-03-24T02:52:31.789 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66080.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85073.log 2026-03-24T02:52:31.790 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.66080.log.gz 2026-03-24T02:52:31.790 INFO:teuthology.orchestra.run.vm07.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50015.log.gz 2026-03-24T02:52:31.791 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82294.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82294.log.gz 2026-03-24T02:52:31.791 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40207.log 2026-03-24T02:52:31.792 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85073.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.85073.log.gz 2026-03-24T02:52:31.792 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49072.log 2026-03-24T02:52:31.792 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40207.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40207.log.gz 2026-03-24T02:52:31.792 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.23082.log 2026-03-24T02:52:31.793 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.49072.log: 84.3% -- replaced with /var/log/ceph/cluster2-client.mirror.49072.log.gz 2026-03-24T02:52:31.793 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26437.log 2026-03-24T02:52:31.793 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.23082.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.23082.log.gz 2026-03-24T02:52:31.794 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47440.log 2026-03-24T02:52:31.802 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.26437.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29513.log 2026-03-24T02:52:31.810 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47440.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47440.log.gz 2026-03-24T02:52:31.810 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42163.log 2026-03-24T02:52:31.811 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29513.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.87256.log 2026-03-24T02:52:31.830 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42163.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.42163.log.gz 2026-03-24T02:52:31.842 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79508.log 2026-03-24T02:52:31.844 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.3.87256.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.3.87256.log.gz 2026-03-24T02:52:31.854 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88669.log 2026-03-24T02:52:31.855 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79508.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.79508.log.gz 2026-03-24T02:52:31.866 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79844.log 2026-03-24T02:52:31.867 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88669.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.88669.log.gz 2026-03-24T02:52:31.874 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35942.log 2026-03-24T02:52:31.883 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79844.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.79844.log.gz 2026-03-24T02:52:31.883 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82197.log 2026-03-24T02:52:31.884 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35942.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.35942.log.gz 2026-03-24T02:52:31.894 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35512.log 2026-03-24T02:52:31.895 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82197.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82197.log.gz 2026-03-24T02:52:31.895 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78583.log 2026-03-24T02:52:31.895 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35512.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.35512.log.gz 2026-03-24T02:52:31.895 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84893.log 2026-03-24T02:52:31.896 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78583.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52906.log 2026-03-24T02:52:31.896 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.78583.log.gz 2026-03-24T02:52:31.896 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84893.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84893.log.gz 2026-03-24T02:52:31.897 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36314.log 2026-03-24T02:52:31.897 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.52906.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.52906.log.gz 2026-03-24T02:52:31.897 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33505.log 2026-03-24T02:52:31.898 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36314.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70451.log 2026-03-24T02:52:31.898 INFO:teuthology.orchestra.run.vm07.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.36314.log.gz 2026-03-24T02:52:31.898 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33505.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33505.log.gz 2026-03-24T02:52:31.898 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48484.log 2026-03-24T02:52:31.899 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.70451.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80671.log 2026-03-24T02:52:31.899 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70451.log.gz 2026-03-24T02:52:31.899 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.48484.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48484.log.gz 2026-03-24T02:52:31.900 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61880.log 2026-03-24T02:52:31.900 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80671.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46421.log 2026-03-24T02:52:31.901 INFO:teuthology.orchestra.run.vm07.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80671.log.gz 2026-03-24T02:52:31.901 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61880.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.61880.log.gz 2026-03-24T02:52:31.902 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88878.log 2026-03-24T02:52:31.902 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46421.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46421.log.gz 2026-03-24T02:52:31.902 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79873.log 2026-03-24T02:52:31.903 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88878.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88878.log.gz 2026-03-24T02:52:31.903 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41529.log 2026-03-24T02:52:31.904 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79873.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88226.log 2026-03-24T02:52:31.904 INFO:teuthology.orchestra.run.vm07.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79873.log.gz 2026-03-24T02:52:31.905 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41529.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54569.log 2026-03-24T02:52:31.907 INFO:teuthology.orchestra.run.vm07.stderr: 96.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41529.log.gz 2026-03-24T02:52:31.914 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88226.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44259.log 2026-03-24T02:52:31.918 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54569.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54569.log.gz 2026-03-24T02:52:31.918 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31857.log 2026-03-24T02:52:31.924 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44259.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44259.log.gz 2026-03-24T02:52:31.930 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60842.log 2026-03-24T02:52:31.931 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31857.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.31857.log.gz 2026-03-24T02:52:31.931 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56424.log 2026-03-24T02:52:31.934 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.60842.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.60842.log.gz 2026-03-24T02:52:31.938 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25631.log 2026-03-24T02:52:31.942 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56424.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56424.log.gz 2026-03-24T02:52:31.942 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46085.log 2026-03-24T02:52:31.943 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.25631.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25631.log.gz 2026-03-24T02:52:31.950 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64969.log 2026-03-24T02:52:31.962 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46085.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45314.log 2026-03-24T02:52:31.963 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64969.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64969.log.gz 2026-03-24T02:52:31.970 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74985.log 2026-03-24T02:52:31.970 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45314.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45314.log.gz 2026-03-24T02:52:31.971 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79537.log 2026-03-24T02:52:31.974 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.74985.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74985.log.gz 2026-03-24T02:52:31.986 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83656.log 2026-03-24T02:52:31.987 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79537.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51374.log 2026-03-24T02:52:31.987 INFO:teuthology.orchestra.run.vm07.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.79537.log.gz 2026-03-24T02:52:31.987 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83656.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83656.log.gz 2026-03-24T02:52:32.002 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56198.log 2026-03-24T02:52:32.002 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.51374.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38688.log 2026-03-24T02:52:32.003 INFO:teuthology.orchestra.run.vm07.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-client.mirror.51374.log.gz 2026-03-24T02:52:32.006 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56198.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56198.log.gz 2026-03-24T02:52:32.014 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82822.log 2026-03-24T02:52:32.015 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.38688.log: 94.1% -- replaced with /var/log/ceph/cluster1-client.mirror.38688.log.gz 2026-03-24T02:52:32.018 INFO:teuthology.orchestra.run.vm07.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46085.log.gz 2026-03-24T02:52:32.019 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60877.log 2026-03-24T02:52:32.019 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82822.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82822.log.gz 2026-03-24T02:52:32.019 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76739.log 2026-03-24T02:52:32.020 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.60877.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.60877.log.gz 2026-03-24T02:52:32.020 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83386.log 2026-03-24T02:52:32.021 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76739.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76739.log.gz 2026-03-24T02:52:32.021 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65147.log 2026-03-24T02:52:32.021 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83386.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83386.log.gz 2026-03-24T02:52:32.021 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.51001.log 2026-03-24T02:52:32.022 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65147.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65147.log.gz 2026-03-24T02:52:32.022 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57050.log 2026-03-24T02:52:32.029 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.2.51001.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35477.log 2026-03-24T02:52:32.034 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.57050.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.57050.log.gz 2026-03-24T02:52:32.034 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26809.log 2026-03-24T02:52:32.038 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35477.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.35477.log.gz 2026-03-24T02:52:32.038 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72682.log 2026-03-24T02:52:32.042 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.26809.log: 62.9% -- replaced with /var/log/ceph/cluster1-client.mirror.26809.log.gz 2026-03-24T02:52:32.046 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87081.log 2026-03-24T02:52:32.050 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72682.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72682.log.gz 2026-03-24T02:52:32.050 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73576.log 2026-03-24T02:52:32.051 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87081.log: 83.7% -- replaced with /var/log/ceph/cluster2-client.mirror.87081.log.gz 2026-03-24T02:52:32.052 INFO:teuthology.orchestra.run.vm07.stderr: 96.0% -- replaced with /var/log/ceph/cluster2-client.mirror.88226.log.gz 2026-03-24T02:52:32.054 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26427.log 2026-03-24T02:52:32.062 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.73576.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73576.log.gz 2026-03-24T02:52:32.062 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39216.log 2026-03-24T02:52:32.070 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.26427.log: 21.7% -- replaced with /var/log/ceph/cluster2-client.mirror.26427.log.gz 2026-03-24T02:52:32.070 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76452.log 2026-03-24T02:52:32.078 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39216.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.39216.log.gz 2026-03-24T02:52:32.078 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64505.log 2026-03-24T02:52:32.079 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.76452.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.76452.log.gz 2026-03-24T02:52:32.086 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73151.log 2026-03-24T02:52:32.094 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.64505.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.64505.log.gz 2026-03-24T02:52:32.095 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58548.log 2026-03-24T02:52:32.096 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.73151.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.73151.log.gz 2026-03-24T02:52:32.102 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37337.log 2026-03-24T02:52:32.110 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.58548.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.58548.log.gz 2026-03-24T02:52:32.110 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84106.log 2026-03-24T02:52:32.111 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37337.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37337.log.gz 2026-03-24T02:52:32.122 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70769.log 2026-03-24T02:52:32.123 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84106.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84106.log.gz 2026-03-24T02:52:32.141 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47510.log 2026-03-24T02:52:32.141 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.70769.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70769.log.gz 2026-03-24T02:52:32.158 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85735.log 2026-03-24T02:52:32.159 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47510.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47510.log.gz 2026-03-24T02:52:32.174 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29167.log 2026-03-24T02:52:32.175 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85735.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.85735.log.gz 2026-03-24T02:52:32.190 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65204.log 2026-03-24T02:52:32.206 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29167.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55490.log 2026-03-24T02:52:32.208 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65204.log: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65204.log.gz 2026-03-24T02:52:32.218 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67166.log 2026-03-24T02:52:32.219 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55490.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55490.log.gz 2026-03-24T02:52:32.234 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81204.log 2026-03-24T02:52:32.235 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.67166.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67166.log.gz 2026-03-24T02:52:32.250 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79915.log 2026-03-24T02:52:32.252 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81204.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81204.log.gz 2026-03-24T02:52:32.266 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80133.log 2026-03-24T02:52:32.268 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79915.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.79915.log.gz 2026-03-24T02:52:32.286 INFO:teuthology.orchestra.run.vm07.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.29513.log.gz 2026-03-24T02:52:32.286 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83061.log 2026-03-24T02:52:32.287 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80133.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.80133.log.gz 2026-03-24T02:52:32.294 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37990.log 2026-03-24T02:52:32.295 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.83061.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83061.log.gz 2026-03-24T02:52:32.314 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26429.log 2026-03-24T02:52:32.317 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.37990.log: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.37990.log.gz 2026-03-24T02:52:32.326 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54840.log 2026-03-24T02:52:32.326 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.26429.log: 77.7% -- replaced with /var/log/ceph/cluster2-client.mirror.26429.log.gz 2026-03-24T02:52:32.342 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82323.log 2026-03-24T02:52:32.343 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54840.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54840.log.gz 2026-03-24T02:52:32.358 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88424.log 2026-03-24T02:52:32.359 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82323.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.82323.log.gz 2026-03-24T02:52:32.370 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87774.log 2026-03-24T02:52:32.372 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88424.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88424.log.gz 2026-03-24T02:52:32.383 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.19296.log 2026-03-24T02:52:32.383 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87774.log: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.87774.log.gz 2026-03-24T02:52:32.394 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80991.log 2026-03-24T02:52:32.395 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/ceph.tmp-client.admin.19296.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.19296.log.gz 2026-03-24T02:52:32.402 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84736.log 2026-03-24T02:52:32.405 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80991.log: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27582.log.gz 2026-03-24T02:52:32.411 INFO:teuthology.orchestra.run.vm07.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80991.log.gz 2026-03-24T02:52:32.411 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53771.log 2026-03-24T02:52:32.412 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84736.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84736.log.gz 2026-03-24T02:52:32.422 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46000.log 2026-03-24T02:52:32.423 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53771.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53771.log.gz 2026-03-24T02:52:32.431 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37093.log 2026-03-24T02:52:32.436 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46000.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78824.log 2026-03-24T02:52:32.438 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37093.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46000.log.gz 2026-03-24T02:52:32.438 INFO:teuthology.orchestra.run.vm07.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.37093.log.gz 2026-03-24T02:52:32.441 INFO:teuthology.orchestra.run.vm07.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.29167.log.gz 2026-03-24T02:52:32.446 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76858.log 2026-03-24T02:52:32.447 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78824.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.78824.log.gz 2026-03-24T02:52:32.454 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86510.log 2026-03-24T02:52:32.466 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76858.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76858.log.gz 2026-03-24T02:52:32.466 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84578.log 2026-03-24T02:52:32.482 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.86510.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89577.log 2026-03-24T02:52:32.483 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84578.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84578.log.gz 2026-03-24T02:52:32.498 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65745.log 2026-03-24T02:52:32.498 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.89577.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.89577.log.gz 2026-03-24T02:52:32.514 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84803.log 2026-03-24T02:52:32.515 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65745.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65745.log.gz 2026-03-24T02:52:32.526 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40149.log 2026-03-24T02:52:32.527 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84803.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84803.log.gz 2026-03-24T02:52:32.534 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60707.log 2026-03-24T02:52:32.546 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.40149.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88464.log 2026-03-24T02:52:32.546 INFO:teuthology.orchestra.run.vm07.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40149.log.gz 2026-03-24T02:52:32.547 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.60707.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60707.log.gz 2026-03-24T02:52:32.558 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60167.log 2026-03-24T02:52:32.559 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88464.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.88464.log.gz 2026-03-24T02:52:32.566 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41552.log 2026-03-24T02:52:32.574 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.60167.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.60167.log.gz 2026-03-24T02:52:32.574 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88053.log 2026-03-24T02:52:32.586 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41552.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37186.log 2026-03-24T02:52:32.586 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.88053.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.88053.log.gz 2026-03-24T02:52:32.598 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78541.log 2026-03-24T02:52:32.599 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.37186.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.37186.log.gz 2026-03-24T02:52:32.606 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84038.log 2026-03-24T02:52:32.614 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78541.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.86510.log.gz 2026-03-24T02:52:32.614 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.78541.log.gz 2026-03-24T02:52:32.626 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42902.log 2026-03-24T02:52:32.627 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84038.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84038.log.gz 2026-03-24T02:52:32.642 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59534.log 2026-03-24T02:52:32.643 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.42902.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.42902.log.gz 2026-03-24T02:52:32.654 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26244.log 2026-03-24T02:52:32.659 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59534.log: 97.2% -- replaced with /var/log/ceph/cluster2-client.mirror.59534.log.gz 2026-03-24T02:52:32.666 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41605.log 2026-03-24T02:52:32.666 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.26244.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.26244.log.gz 2026-03-24T02:52:32.678 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69677.log 2026-03-24T02:52:32.679 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.41605.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.41605.log.gz 2026-03-24T02:52:32.694 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79631.log 2026-03-24T02:52:32.698 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.69677.log: 96.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69677.log.gz 2026-03-24T02:52:32.710 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60983.log 2026-03-24T02:52:32.711 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79631.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.79631.log.gz 2026-03-24T02:52:32.722 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54141.log 2026-03-24T02:52:32.722 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.60983.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.60983.log.gz 2026-03-24T02:52:32.730 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31485.log 2026-03-24T02:52:32.732 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.54141.log: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26437.log.gz 2026-03-24T02:52:32.734 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56969.log 2026-03-24T02:52:32.735 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.31485.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.31485.log.gz 2026-03-24T02:52:32.735 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81681.log 2026-03-24T02:52:32.735 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56969.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.56969.log.gz 2026-03-24T02:52:32.736 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79323.log 2026-03-24T02:52:32.736 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81681.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.81681.log.gz 2026-03-24T02:52:32.737 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49894.log 2026-03-24T02:52:32.737 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79323.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.79323.log.gz 2026-03-24T02:52:32.738 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80764.log 2026-03-24T02:52:32.739 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.49894.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54141.log.gz 2026-03-24T02:52:32.739 INFO:teuthology.orchestra.run.vm07.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49894.log.gz 2026-03-24T02:52:32.739 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73252.log 2026-03-24T02:52:32.739 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80764.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80764.log.gz 2026-03-24T02:52:32.739 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84151.log 2026-03-24T02:52:32.740 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.73252.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73252.log.gz 2026-03-24T02:52:32.740 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44170.log 2026-03-24T02:52:32.740 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84151.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84151.log.gz 2026-03-24T02:52:32.741 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41928.log 2026-03-24T02:52:32.741 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44170.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85480.log 2026-03-24T02:52:32.744 INFO:teuthology.orchestra.run.vm07.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44170.log.gz 2026-03-24T02:52:32.744 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41928.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41928.log.gz 2026-03-24T02:52:32.744 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72825.log 2026-03-24T02:52:32.745 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85480.log: 63.2% -- replaced with /var/log/ceph/cluster1-client.mirror.85480.log.gz 2026-03-24T02:52:32.745 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43803.log 2026-03-24T02:52:32.746 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72825.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.72825.log.gz 2026-03-24T02:52:32.746 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.42584.log 2026-03-24T02:52:32.749 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.43803.log: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43803.log.gz 2026-03-24T02:52:32.749 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73106.log 2026-03-24T02:52:32.750 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.0.42584.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53259.log 2026-03-24T02:52:32.751 INFO:teuthology.orchestra.run.vm07.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.0.42584.log.gz 2026-03-24T02:52:32.751 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.73106.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69352.log 2026-03-24T02:52:32.751 INFO:teuthology.orchestra.run.vm07.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.73106.log.gz 2026-03-24T02:52:32.752 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53259.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83341.log 2026-03-24T02:52:32.755 INFO:teuthology.orchestra.run.vm07.stderr: 96.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53259.log.gz 2026-03-24T02:52:32.756 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.69352.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69352.log.gz 2026-03-24T02:52:32.756 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46330.log 2026-03-24T02:52:32.756 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83341.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.83341.log.gz 2026-03-24T02:52:32.756 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83251.log 2026-03-24T02:52:32.757 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46330.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46330.log.gz 2026-03-24T02:52:32.757 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61972.log 2026-03-24T02:52:32.757 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83251.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.83251.log.gz 2026-03-24T02:52:32.758 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38116.log 2026-03-24T02:52:32.758 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61972.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39524.log 2026-03-24T02:52:32.759 INFO:teuthology.orchestra.run.vm07.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61972.log.gz 2026-03-24T02:52:32.774 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38116.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52871.log 2026-03-24T02:52:32.775 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.39524.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88076.log 2026-03-24T02:52:32.777 INFO:teuthology.orchestra.run.vm07.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.39524.log.gz 2026-03-24T02:52:32.778 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.52871.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52871.log.gz 2026-03-24T02:52:32.778 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67303.log 2026-03-24T02:52:32.778 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.88076.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88076.log.gz 2026-03-24T02:52:32.786 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36721.log 2026-03-24T02:52:32.790 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.67303.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.67303.log.gz 2026-03-24T02:52:32.790 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29869.log 2026-03-24T02:52:32.798 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36721.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.36721.log.gz 2026-03-24T02:52:32.798 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66490.log 2026-03-24T02:52:32.799 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.29869.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29869.log.gz 2026-03-24T02:52:32.800 INFO:teuthology.orchestra.run.vm07.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.41552.log.gz 2026-03-24T02:52:32.810 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60682.log 2026-03-24T02:52:32.811 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66490.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66490.log.gz 2026-03-24T02:52:32.811 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64004.log 2026-03-24T02:52:32.812 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.60682.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.60682.log.gz 2026-03-24T02:52:32.812 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85515.log 2026-03-24T02:52:32.816 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64004.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.64004.log.gz 2026-03-24T02:52:32.826 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66606.log 2026-03-24T02:52:32.827 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.85515.log: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.85515.log.gz 2026-03-24T02:52:32.838 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83132.log 2026-03-24T02:52:32.839 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66606.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.66606.log.gz 2026-03-24T02:52:32.850 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32664.log 2026-03-24T02:52:32.851 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.83132.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83132.log.gz 2026-03-24T02:52:32.854 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38116.log.gz 2026-03-24T02:52:32.854 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77244.log 2026-03-24T02:52:32.855 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32664.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32664.log.gz 2026-03-24T02:52:32.855 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77646.log 2026-03-24T02:52:32.855 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.77244.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.77244.log.gz 2026-03-24T02:52:32.856 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28952.log 2026-03-24T02:52:32.856 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77646.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55370.log 2026-03-24T02:52:32.857 INFO:teuthology.orchestra.run.vm07.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77646.log.gz 2026-03-24T02:52:32.857 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.28952.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.28952.log.gz 2026-03-24T02:52:32.857 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86920.log 2026-03-24T02:52:32.858 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55370.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.55370.log.gz 2026-03-24T02:52:32.858 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52657.log 2026-03-24T02:52:32.859 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.86920.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.86920.log.gz 2026-03-24T02:52:32.859 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38446.log 2026-03-24T02:52:32.860 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.52657.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77069.log 2026-03-24T02:52:32.867 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38446.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38446.log.gz 2026-03-24T02:52:32.868 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81113.log 2026-03-24T02:52:32.868 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82538.log 2026-03-24T02:52:32.868 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77069.log: /var/log/ceph/cluster2-client.mirror.81113.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61775.log 2026-03-24T02:52:32.869 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38012.log 2026-03-24T02:52:32.869 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82538.log: /var/log/ceph/cluster2-client.mirror.61775.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58189.log 2026-03-24T02:52:32.869 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38012.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66571.log 2026-03-24T02:52:32.870 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.58189.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32420.log 2026-03-24T02:52:32.878 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66571.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81113.log.gz 2026-03-24T02:52:32.878 INFO:teuthology.orchestra.run.vm07.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.38012.log.gz 2026-03-24T02:52:32.878 INFO:teuthology.orchestra.run.vm07.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52657.log.gz 2026-03-24T02:52:32.878 INFO:teuthology.orchestra.run.vm07.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77069.log.gz 2026-03-24T02:52:32.878 INFO:teuthology.orchestra.run.vm07.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82538.log.gz 2026-03-24T02:52:32.878 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78506.log 2026-03-24T02:52:32.878 INFO:teuthology.orchestra.run.vm07.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58189.log.gz 2026-03-24T02:52:32.879 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.32420.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32420.log.gz 2026-03-24T02:52:32.879 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41716.log 2026-03-24T02:52:32.879 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.78506.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69317.log 2026-03-24T02:52:32.880 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.78506.log.gz 2026-03-24T02:52:32.880 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.41716.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.41716.log.gz 2026-03-24T02:52:32.880 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64438.log 2026-03-24T02:52:32.881 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.69317.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.69317.log.gz 2026-03-24T02:52:32.881 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26843.log 2026-03-24T02:52:32.882 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64438.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64438.log.gz 2026-03-24T02:52:32.882 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79802.log 2026-03-24T02:52:32.883 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.26843.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66571.log.gz 2026-03-24T02:52:32.883 INFO:teuthology.orchestra.run.vm07.stderr: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.26843.log.gz 2026-03-24T02:52:32.883 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51644.log 2026-03-24T02:52:32.884 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79802.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80580.log 2026-03-24T02:52:32.884 INFO:teuthology.orchestra.run.vm07.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.79802.log.gz 2026-03-24T02:52:32.885 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.51644.log: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51644.log.gz 2026-03-24T02:52:32.885 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59783.log 2026-03-24T02:52:32.886 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80580.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.80580.log.gz 2026-03-24T02:52:32.886 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58069.log 2026-03-24T02:52:32.886 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.59783.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.61775.log.gz 2026-03-24T02:52:32.886 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.59783.log.gz 2026-03-24T02:52:32.886 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85208.log 2026-03-24T02:52:32.887 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.58069.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.58069.log.gz 2026-03-24T02:52:32.887 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80906.log 2026-03-24T02:52:32.888 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.85208.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.85208.log.gz 2026-03-24T02:52:32.888 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43846.log 2026-03-24T02:52:32.889 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80906.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.80906.log.gz 2026-03-24T02:52:32.889 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26092.log 2026-03-24T02:52:32.892 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.43846.log: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43846.log.gz 2026-03-24T02:52:32.892 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84848.log 2026-03-24T02:52:32.892 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.26092.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.26092.log.gz 2026-03-24T02:52:32.893 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34140.log 2026-03-24T02:52:32.893 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.84848.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84848.log.gz 2026-03-24T02:52:32.893 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43682.log 2026-03-24T02:52:32.894 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34140.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.34140.log.gz 2026-03-24T02:52:32.894 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76283.log 2026-03-24T02:52:32.895 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43682.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.43682.log.gz 2026-03-24T02:52:32.895 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72175.log 2026-03-24T02:52:32.895 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76283.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76283.log.gz 2026-03-24T02:52:32.896 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88540.log 2026-03-24T02:52:32.896 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72175.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72175.log.gz 2026-03-24T02:52:32.896 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81326.log 2026-03-24T02:52:32.897 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.88540.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88540.log.gz 2026-03-24T02:52:32.897 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25728.log 2026-03-24T02:52:32.898 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81326.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81326.log.gz 2026-03-24T02:52:32.898 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34877.log 2026-03-24T02:52:32.898 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.admin.25728.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25728.log.gz 2026-03-24T02:52:32.898 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60534.log 2026-03-24T02:52:32.899 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34877.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.34877.log.gz 2026-03-24T02:52:32.899 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36593.log 2026-03-24T02:52:32.900 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.60534.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.60534.log.gz 2026-03-24T02:52:32.900 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87148.log 2026-03-24T02:52:32.900 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36593.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.36593.log.gz 2026-03-24T02:52:32.901 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71283.log 2026-03-24T02:52:32.910 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87148.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38886.log 2026-03-24T02:52:32.919 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.71283.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.71283.log.gz 2026-03-24T02:52:32.919 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78036.log 2026-03-24T02:52:32.926 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.38886.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36535.log 2026-03-24T02:52:32.934 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.78036.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68218.log 2026-03-24T02:52:32.935 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.36535.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.36535.log.gz 2026-03-24T02:52:32.935 INFO:teuthology.orchestra.run.vm07.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.78036.log.gz 2026-03-24T02:52:32.942 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61018.log 2026-03-24T02:52:32.950 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68218.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.68218.log.gz 2026-03-24T02:52:32.950 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45013.log 2026-03-24T02:52:32.951 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.61018.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.61018.log.gz 2026-03-24T02:52:32.954 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82915.log 2026-03-24T02:52:32.962 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45013.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77624.log 2026-03-24T02:52:32.966 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.82915.log: 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.2.51001.log.gz 2026-03-24T02:52:32.966 INFO:teuthology.orchestra.run.vm07.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82915.log.gz 2026-03-24T02:52:32.966 INFO:teuthology.orchestra.run.vm07.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.45013.log.gz 2026-03-24T02:52:32.967 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45647.log 2026-03-24T02:52:32.968 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.77624.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.77624.log.gz 2026-03-24T02:52:32.968 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46557.log 2026-03-24T02:52:32.968 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.45647.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45647.log.gz 2026-03-24T02:52:32.969 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87178.log 2026-03-24T02:52:32.969 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46557.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46557.log.gz 2026-03-24T02:52:32.969 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46444.log 2026-03-24T02:52:32.970 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.87178.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.87178.log.gz 2026-03-24T02:52:32.970 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81233.log 2026-03-24T02:52:32.978 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46444.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71641.log 2026-03-24T02:52:32.986 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81233.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81233.log.gz 2026-03-24T02:52:32.987 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88908.log 2026-03-24T02:52:32.987 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.71641.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71641.log.gz 2026-03-24T02:52:32.988 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56693.log 2026-03-24T02:52:32.988 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.88908.log: 78.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88908.log.gz 2026-03-24T02:52:32.994 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65625.log 2026-03-24T02:52:33.000 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.56693.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56693.log.gz 2026-03-24T02:52:33.001 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33133.log 2026-03-24T02:52:33.010 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.65625.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63687.log 2026-03-24T02:52:33.010 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65625.log.gz 2026-03-24T02:52:33.011 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58903.log 2026-03-24T02:52:33.015 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33133.log: /var/log/ceph/cluster2-client.mirror.63687.log: 97.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63687.log.gz 2026-03-24T02:52:33.018 INFO:teuthology.orchestra.run.vm07.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.33133.log.gz 2026-03-24T02:52:33.026 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46689.log 2026-03-24T02:52:33.027 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.58903.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.58903.log.gz 2026-03-24T02:52:33.038 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33447.log 2026-03-24T02:52:33.039 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.46689.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46689.log.gz 2026-03-24T02:52:33.040 INFO:teuthology.orchestra.run.vm07.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.87148.log.gz 2026-03-24T02:52:33.046 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39335.log 2026-03-24T02:52:33.054 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.33447.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33447.log.gz 2026-03-24T02:52:33.058 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76420.log 2026-03-24T02:52:33.059 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.39335.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39335.log.gz 2026-03-24T02:52:33.069 INFO:teuthology.orchestra.run.vm07.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38886.log.gz 2026-03-24T02:52:33.069 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68675.log 2026-03-24T02:52:33.070 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76420.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76420.log.gz 2026-03-24T02:52:33.070 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87378.log 2026-03-24T02:52:33.071 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.68675.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35884.log 2026-03-24T02:52:33.072 INFO:teuthology.orchestra.run.vm07.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.68675.log.gz 2026-03-24T02:52:33.072 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.87378.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.87378.log.gz 2026-03-24T02:52:33.072 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79824.log 2026-03-24T02:52:33.073 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.35884.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.35884.log.gz 2026-03-24T02:52:33.073 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68848.log 2026-03-24T02:52:33.073 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.79824.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44691.log 2026-03-24T02:52:33.074 INFO:teuthology.orchestra.run.vm07.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79824.log.gz 2026-03-24T02:52:33.074 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.68848.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.68848.log.gz 2026-03-24T02:52:33.074 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47161.log 2026-03-24T02:52:33.075 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.44691.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44691.log.gz 2026-03-24T02:52:33.075 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81794.log 2026-03-24T02:52:33.076 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.47161.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47161.log.gz 2026-03-24T02:52:33.076 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43955.log 2026-03-24T02:52:33.077 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81794.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72032.log 2026-03-24T02:52:33.078 INFO:teuthology.orchestra.run.vm07.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81794.log.gz 2026-03-24T02:52:33.078 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.43955.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85534.log 2026-03-24T02:52:33.081 INFO:teuthology.orchestra.run.vm07.stderr: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.43955.log.gz 2026-03-24T02:52:33.081 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.72032.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72032.log.gz 2026-03-24T02:52:33.081 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77564.log 2026-03-24T02:52:33.082 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.85534.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.85534.log.gz 2026-03-24T02:52:33.082 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64572.log 2026-03-24T02:52:33.083 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.77564.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77564.log.gz 2026-03-24T02:52:33.084 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mon.a.log 2026-03-24T02:52:33.084 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.64572.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74752.log 2026-03-24T02:52:33.085 INFO:teuthology.orchestra.run.vm07.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64572.log.gz 2026-03-24T02:52:33.086 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83971.log 2026-03-24T02:52:33.092 INFO:teuthology.orchestra.run.vm07.stderr: 96.1% -- replaced with /var/log/ceph/cluster2-client.mirror.46444.log.gz 2026-03-24T02:52:33.094 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.74752.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74752.log.gz 2026-03-24T02:52:33.102 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78225.log 2026-03-24T02:52:33.103 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.83971.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.83971.log.gz 2026-03-24T02:52:33.110 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80374.log 2026-03-24T02:52:33.111 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.78225.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.78225.log.gz 2026-03-24T02:52:33.122 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81184.log 2026-03-24T02:52:33.124 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80374.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80374.log.gz 2026-03-24T02:52:33.126 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76087.log 2026-03-24T02:52:33.127 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.81184.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81184.log.gz 2026-03-24T02:52:33.127 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89064.log 2026-03-24T02:52:33.127 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.76087.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76087.log.gz 2026-03-24T02:52:33.127 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80081.log 2026-03-24T02:52:33.128 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.89064.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.89064.log.gz 2026-03-24T02:52:33.128 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28798.log 2026-03-24T02:52:33.129 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.80081.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.80081.log.gz 2026-03-24T02:52:33.129 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34468.log 2026-03-24T02:52:33.130 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.28798.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.28798.log.gz 2026-03-24T02:52:33.130 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87753.log 2026-03-24T02:52:33.142 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.34468.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29468.log 2026-03-24T02:52:33.143 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87753.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.87753.log.gz 2026-03-24T02:52:33.154 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44412.log 2026-03-24T02:52:33.157 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.29468.log: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.29468.log.gz 2026-03-24T02:52:33.166 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53327.log 2026-03-24T02:52:33.169 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.44412.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.44412.log.gz 2026-03-24T02:52:33.182 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66260.log 2026-03-24T02:52:33.182 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.53327.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.53327.log.gz 2026-03-24T02:52:33.194 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87480.log 2026-03-24T02:52:33.195 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.66260.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.54459.log 2026-03-24T02:52:33.199 INFO:teuthology.orchestra.run.vm07.stderr: 97.2% -- replaced with /var/log/ceph/cluster2-client.mirror.66260.log.gz 2026-03-24T02:52:33.200 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.87480.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.87480.log.gz 2026-03-24T02:52:33.214 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41574.log 2026-03-24T02:52:33.220 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.2.54459.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.2.54459.log.gz 2026-03-24T02:52:33.230 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55057.log 2026-03-24T02:52:33.231 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.admin.41574.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.admin.41574.log.gz 2026-03-24T02:52:33.242 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43975.log 2026-03-24T02:52:33.243 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-client.mirror.55057.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55057.log.gz 2026-03-24T02:52:33.256 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2026-03-24T02:52:33.259 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster1-client.mirror.43975.log: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43975.log.gz 2026-03-24T02:52:33.333 INFO:teuthology.orchestra.run.vm07.stderr:/var/log/ceph/cluster2-osd.2.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.34468.log.gz 2026-03-24T02:52:33.931 INFO:teuthology.orchestra.run.vm07.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2026-03-24T02:52:58.519 INFO:teuthology.orchestra.run.vm06.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2026-03-24T02:53:01.923 INFO:teuthology.orchestra.run.vm06.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2026-03-24T02:53:08.431 INFO:teuthology.orchestra.run.vm06.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2026-03-24T02:53:08.433 INFO:teuthology.orchestra.run.vm06.stderr: 2026-03-24T02:53:08.433 INFO:teuthology.orchestra.run.vm06.stderr:real 0m43.049s 2026-03-24T02:53:08.433 INFO:teuthology.orchestra.run.vm06.stderr:user 1m38.714s 2026-03-24T02:53:08.433 INFO:teuthology.orchestra.run.vm06.stderr:sys 0m9.446s 2026-03-24T02:53:17.611 INFO:teuthology.orchestra.run.vm07.stderr: 94.8% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2026-03-24T02:53:22.763 INFO:teuthology.orchestra.run.vm07.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2026-03-24T02:53:29.125 INFO:teuthology.orchestra.run.vm07.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2026-03-24T02:53:29.126 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-24T02:53:29.126 INFO:teuthology.orchestra.run.vm07.stderr:real 1m3.734s 2026-03-24T02:53:29.126 INFO:teuthology.orchestra.run.vm07.stderr:user 2m40.089s 2026-03-24T02:53:29.126 INFO:teuthology.orchestra.run.vm07.stderr:sys 0m14.391s 2026-03-24T02:53:29.127 INFO:tasks.ceph:Archiving logs... 2026-03-24T02:53:29.127 DEBUG:teuthology.misc:Transferring archived files from vm06:/var/log/ceph to /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3539/remote/vm06/log 2026-03-24T02:53:29.127 DEBUG:teuthology.orchestra.run.vm06:> sudo tar c -f - -C /var/log/ceph -- . 2026-03-24T02:53:33.815 DEBUG:teuthology.misc:Transferring archived files from vm07:/var/log/ceph to /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3539/remote/vm07/log 2026-03-24T02:53:33.815 DEBUG:teuthology.orchestra.run.vm07:> sudo tar c -f - -C /var/log/ceph -- . 2026-03-24T02:53:41.278 DEBUG:teuthology.run_tasks:Unwinding manager install 2026-03-24T02:53:41.283 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2026-03-24T02:53:41.283 DEBUG:teuthology.orchestra.run.vm06:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-03-24T02:53:41.284 DEBUG:teuthology.orchestra.run.vm07:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-03-24T02:53:41.390 INFO:teuthology.task.install.deb:Removing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on Debian system. 2026-03-24T02:53:41.390 DEBUG:teuthology.orchestra.run.vm06:> for d in ceph cephadm ceph-mds ceph-mgr ceph-common ceph-fuse ceph-test ceph-volume radosgw python3-rados python3-rgw python3-cephfs python3-rbd libcephfs2 libcephfs-dev librados2 librbd1 rbd-fuse rbd-mirror ; do sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" purge $d || true ; done 2026-03-24T02:53:41.396 INFO:teuthology.task.install.deb:Removing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on Debian system. 2026-03-24T02:53:41.397 DEBUG:teuthology.orchestra.run.vm07:> for d in ceph cephadm ceph-mds ceph-mgr ceph-common ceph-fuse ceph-test ceph-volume radosgw python3-rados python3-rgw python3-cephfs python3-rbd libcephfs2 libcephfs-dev librados2 librbd1 rbd-fuse rbd-mirror ; do sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" purge $d || true ; done 2026-03-24T02:53:41.491 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:53:41.553 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:53:41.654 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:53:41.654 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:53:41.727 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:53:41.728 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:53:41.771 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:53:41.772 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mon kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libpmemobj1 2026-03-24T02:53:41.772 INFO:teuthology.orchestra.run.vm07.stdout: libsgutils2-2 sg3-utils sg3-utils-udev 2026-03-24T02:53:41.772 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:53:41.791 INFO:teuthology.orchestra.run.vm07.stdout:The following packages will be REMOVED: 2026-03-24T02:53:41.792 INFO:teuthology.orchestra.run.vm07.stdout: ceph* 2026-03-24T02:53:41.849 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:53:41.849 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mon kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libpmemobj1 2026-03-24T02:53:41.849 INFO:teuthology.orchestra.run.vm06.stdout: libsgutils2-2 sg3-utils sg3-utils-udev 2026-03-24T02:53:41.849 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:53:41.865 INFO:teuthology.orchestra.run.vm06.stdout:The following packages will be REMOVED: 2026-03-24T02:53:41.865 INFO:teuthology.orchestra.run.vm06.stdout: ceph* 2026-03-24T02:53:41.992 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 1 to remove and 44 not upgraded. 2026-03-24T02:53:41.992 INFO:teuthology.orchestra.run.vm07.stdout:After this operation, 47.1 kB disk space will be freed. 2026-03-24T02:53:42.042 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 126157 files and directories currently installed.) 2026-03-24T02:53:42.044 INFO:teuthology.orchestra.run.vm07.stdout:Removing ceph (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:42.048 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 1 to remove and 44 not upgraded. 2026-03-24T02:53:42.048 INFO:teuthology.orchestra.run.vm06.stdout:After this operation, 47.1 kB disk space will be freed. 2026-03-24T02:53:42.099 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 126157 files and directories currently installed.) 2026-03-24T02:53:42.101 INFO:teuthology.orchestra.run.vm06.stdout:Removing ceph (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:43.188 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:53:43.191 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:53:43.221 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:53:43.226 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:53:43.392 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:53:43.393 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:53:43.405 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:53:43.406 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:53:43.520 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:53:43.520 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mon kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libpmemobj1 2026-03-24T02:53:43.521 INFO:teuthology.orchestra.run.vm07.stdout: libsgutils2-2 python-asyncssh-doc python3-asyncssh sg3-utils sg3-utils-udev 2026-03-24T02:53:43.521 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:53:43.537 INFO:teuthology.orchestra.run.vm07.stdout:The following packages will be REMOVED: 2026-03-24T02:53:43.538 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-cephadm* cephadm* 2026-03-24T02:53:43.569 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:53:43.569 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mon kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libpmemobj1 2026-03-24T02:53:43.569 INFO:teuthology.orchestra.run.vm06.stdout: libsgutils2-2 python-asyncssh-doc python3-asyncssh sg3-utils sg3-utils-udev 2026-03-24T02:53:43.569 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:53:43.578 INFO:teuthology.orchestra.run.vm06.stdout:The following packages will be REMOVED: 2026-03-24T02:53:43.578 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-cephadm* cephadm* 2026-03-24T02:53:43.757 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 2 to remove and 44 not upgraded. 2026-03-24T02:53:43.757 INFO:teuthology.orchestra.run.vm07.stdout:After this operation, 2177 kB disk space will be freed. 2026-03-24T02:53:43.813 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 2 to remove and 44 not upgraded. 2026-03-24T02:53:43.813 INFO:teuthology.orchestra.run.vm06.stdout:After this operation, 2177 kB disk space will be freed. 2026-03-24T02:53:43.884 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 126155 files and directories currently installed.) 2026-03-24T02:53:43.885 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 126155 files and directories currently installed.) 2026-03-24T02:53:43.886 INFO:teuthology.orchestra.run.vm07.stdout:Removing ceph-mgr-cephadm (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:43.887 INFO:teuthology.orchestra.run.vm06.stdout:Removing ceph-mgr-cephadm (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:44.099 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-cephadm, directory '/usr/share/ceph/mgr/cephadm/services' not empty so not removed 2026-03-24T02:53:44.108 INFO:teuthology.orchestra.run.vm07.stdout:Removing cephadm (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:44.109 INFO:teuthology.orchestra.run.vm06.stdout:Removing cephadm (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:44.138 INFO:teuthology.orchestra.run.vm07.stdout:Looking for files to backup/remove ... 2026-03-24T02:53:44.138 INFO:teuthology.orchestra.run.vm06.stdout:Looking for files to backup/remove ... 2026-03-24T02:53:44.139 INFO:teuthology.orchestra.run.vm06.stdout:Not backing up/removing `/var/lib/cephadm', it matches ^/var/.*. 2026-03-24T02:53:44.140 INFO:teuthology.orchestra.run.vm07.stdout:Not backing up/removing `/var/lib/cephadm', it matches ^/var/.*. 2026-03-24T02:53:44.141 INFO:teuthology.orchestra.run.vm06.stdout:Removing user `cephadm' ... 2026-03-24T02:53:44.141 INFO:teuthology.orchestra.run.vm06.stdout:Warning: group `nogroup' has no more members. 2026-03-24T02:53:44.142 INFO:teuthology.orchestra.run.vm07.stdout:Removing user `cephadm' ... 2026-03-24T02:53:44.142 INFO:teuthology.orchestra.run.vm07.stdout:Warning: group `nogroup' has no more members. 2026-03-24T02:53:44.154 INFO:teuthology.orchestra.run.vm06.stdout:Done. 2026-03-24T02:53:44.160 INFO:teuthology.orchestra.run.vm07.stdout:Done. 2026-03-24T02:53:44.176 INFO:teuthology.orchestra.run.vm06.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-03-24T02:53:44.180 INFO:teuthology.orchestra.run.vm07.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-03-24T02:53:44.273 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 126069 files and directories currently installed.) 2026-03-24T02:53:44.275 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 126069 files and directories currently installed.) 2026-03-24T02:53:44.276 INFO:teuthology.orchestra.run.vm07.stdout:Purging configuration files for cephadm (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:44.277 INFO:teuthology.orchestra.run.vm06.stdout:Purging configuration files for cephadm (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:45.385 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:53:45.422 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:53:45.425 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:53:45.458 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:53:45.589 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:53:45.589 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:53:45.616 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:53:45.616 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:53:45.716 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:53:45.716 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mon kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libpmemobj1 2026-03-24T02:53:45.717 INFO:teuthology.orchestra.run.vm06.stdout: libsgutils2-2 python-asyncssh-doc python3-asyncssh sg3-utils sg3-utils-udev 2026-03-24T02:53:45.717 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:53:45.731 INFO:teuthology.orchestra.run.vm06.stdout:The following packages will be REMOVED: 2026-03-24T02:53:45.732 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mds* 2026-03-24T02:53:45.751 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:53:45.751 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mon kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libpmemobj1 2026-03-24T02:53:45.751 INFO:teuthology.orchestra.run.vm07.stdout: libsgutils2-2 python-asyncssh-doc python3-asyncssh sg3-utils sg3-utils-udev 2026-03-24T02:53:45.751 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:53:45.759 INFO:teuthology.orchestra.run.vm07.stdout:The following packages will be REMOVED: 2026-03-24T02:53:45.760 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mds* 2026-03-24T02:53:45.901 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 1 to remove and 44 not upgraded. 2026-03-24T02:53:45.901 INFO:teuthology.orchestra.run.vm06.stdout:After this operation, 6851 kB disk space will be freed. 2026-03-24T02:53:45.913 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 1 to remove and 44 not upgraded. 2026-03-24T02:53:45.913 INFO:teuthology.orchestra.run.vm07.stdout:After this operation, 6851 kB disk space will be freed. 2026-03-24T02:53:45.937 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 126069 files and directories currently installed.) 2026-03-24T02:53:45.940 INFO:teuthology.orchestra.run.vm06.stdout:Removing ceph-mds (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:45.949 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 126069 files and directories currently installed.) 2026-03-24T02:53:45.951 INFO:teuthology.orchestra.run.vm07.stdout:Removing ceph-mds (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:46.349 INFO:teuthology.orchestra.run.vm06.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-03-24T02:53:46.365 INFO:teuthology.orchestra.run.vm07.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-03-24T02:53:46.445 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 126061 files and directories currently installed.) 2026-03-24T02:53:46.447 INFO:teuthology.orchestra.run.vm06.stdout:Purging configuration files for ceph-mds (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:46.458 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 126061 files and directories currently installed.) 2026-03-24T02:53:46.460 INFO:teuthology.orchestra.run.vm07.stdout:Purging configuration files for ceph-mds (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:47.802 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:53:47.835 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:53:47.851 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:53:47.886 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:53:48.001 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:53:48.001 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:53:48.055 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:53:48.055 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:53:48.116 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:53:48.117 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core ceph-mon kpartx libboost-iostreams1.74.0 2026-03-24T02:53:48.117 INFO:teuthology.orchestra.run.vm07.stdout: libboost-thread1.74.0 libpmemobj1 libsgutils2-2 python-asyncssh-doc 2026-03-24T02:53:48.117 INFO:teuthology.orchestra.run.vm07.stdout: python3-asyncssh python3-cachetools python3-cheroot python3-cherrypy3 2026-03-24T02:53:48.117 INFO:teuthology.orchestra.run.vm07.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:53:48.117 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:53:48.117 INFO:teuthology.orchestra.run.vm07.stdout: python3-kubernetes python3-natsort python3-portend python3-psutil 2026-03-24T02:53:48.117 INFO:teuthology.orchestra.run.vm07.stdout: python3-repoze.lru python3-requests-oauthlib python3-routes python3-rsa 2026-03-24T02:53:48.117 INFO:teuthology.orchestra.run.vm07.stdout: python3-simplejson python3-sklearn python3-sklearn-lib python3-tempora 2026-03-24T02:53:48.117 INFO:teuthology.orchestra.run.vm07.stdout: python3-threadpoolctl python3-webob python3-websocket python3-zc.lockfile 2026-03-24T02:53:48.117 INFO:teuthology.orchestra.run.vm07.stdout: sg3-utils sg3-utils-udev 2026-03-24T02:53:48.117 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:53:48.128 INFO:teuthology.orchestra.run.vm07.stdout:The following packages will be REMOVED: 2026-03-24T02:53:48.128 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr* ceph-mgr-dashboard* ceph-mgr-diskprediction-local* 2026-03-24T02:53:48.128 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-k8sevents* 2026-03-24T02:53:48.191 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:53:48.191 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core ceph-mon kpartx libboost-iostreams1.74.0 2026-03-24T02:53:48.191 INFO:teuthology.orchestra.run.vm06.stdout: libboost-thread1.74.0 libpmemobj1 libsgutils2-2 python-asyncssh-doc 2026-03-24T02:53:48.191 INFO:teuthology.orchestra.run.vm06.stdout: python3-asyncssh python3-cachetools python3-cheroot python3-cherrypy3 2026-03-24T02:53:48.191 INFO:teuthology.orchestra.run.vm06.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:53:48.191 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:53:48.191 INFO:teuthology.orchestra.run.vm06.stdout: python3-kubernetes python3-natsort python3-portend python3-psutil 2026-03-24T02:53:48.191 INFO:teuthology.orchestra.run.vm06.stdout: python3-repoze.lru python3-requests-oauthlib python3-routes python3-rsa 2026-03-24T02:53:48.191 INFO:teuthology.orchestra.run.vm06.stdout: python3-simplejson python3-sklearn python3-sklearn-lib python3-tempora 2026-03-24T02:53:48.191 INFO:teuthology.orchestra.run.vm06.stdout: python3-threadpoolctl python3-webob python3-websocket python3-zc.lockfile 2026-03-24T02:53:48.191 INFO:teuthology.orchestra.run.vm06.stdout: sg3-utils sg3-utils-udev 2026-03-24T02:53:48.191 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:53:48.199 INFO:teuthology.orchestra.run.vm06.stdout:The following packages will be REMOVED: 2026-03-24T02:53:48.200 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr* ceph-mgr-dashboard* ceph-mgr-diskprediction-local* 2026-03-24T02:53:48.200 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-k8sevents* 2026-03-24T02:53:48.379 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 4 to remove and 44 not upgraded. 2026-03-24T02:53:48.379 INFO:teuthology.orchestra.run.vm06.stdout:After this operation, 219 MB disk space will be freed. 2026-03-24T02:53:48.379 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 4 to remove and 44 not upgraded. 2026-03-24T02:53:48.379 INFO:teuthology.orchestra.run.vm07.stdout:After this operation, 219 MB disk space will be freed. 2026-03-24T02:53:48.421 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 126061 files and directories currently installed.) 2026-03-24T02:53:48.423 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 126061 files and directories currently installed.) 2026-03-24T02:53:48.423 INFO:teuthology.orchestra.run.vm06.stdout:Removing ceph-mgr-k8sevents (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:48.426 INFO:teuthology.orchestra.run.vm07.stdout:Removing ceph-mgr-k8sevents (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:48.445 INFO:teuthology.orchestra.run.vm06.stdout:Removing ceph-mgr-diskprediction-local (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:48.448 INFO:teuthology.orchestra.run.vm07.stdout:Removing ceph-mgr-diskprediction-local (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:48.460 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-diskprediction-local, directory '/usr/share/ceph/mgr/diskprediction_local' not empty so not removed 2026-03-24T02:53:48.471 INFO:teuthology.orchestra.run.vm06.stdout:Removing ceph-mgr-dashboard (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:48.472 INFO:teuthology.orchestra.run.vm07.stdout:Removing ceph-mgr-dashboard (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:48.530 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-dashboard, directory '/usr/share/ceph/mgr/dashboard/services/auth' not empty so not removed 2026-03-24T02:53:48.530 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-dashboard, directory '/usr/share/ceph/mgr/dashboard/plugins' not empty so not removed 2026-03-24T02:53:48.530 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-dashboard, directory '/usr/share/ceph/mgr/dashboard/model' not empty so not removed 2026-03-24T02:53:48.530 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-dashboard, directory '/usr/share/ceph/mgr/dashboard/controllers' not empty so not removed 2026-03-24T02:53:48.530 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-dashboard, directory '/usr/share/ceph/mgr/dashboard/api' not empty so not removed 2026-03-24T02:53:48.540 INFO:teuthology.orchestra.run.vm07.stdout:Removing ceph-mgr (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:48.540 INFO:teuthology.orchestra.run.vm06.stdout:Removing ceph-mgr (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:49.008 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 124278 files and directories currently installed.) 2026-03-24T02:53:49.010 INFO:teuthology.orchestra.run.vm07.stdout:Purging configuration files for ceph-mgr (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:49.021 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 124279 files and directories currently installed.) 2026-03-24T02:53:49.023 INFO:teuthology.orchestra.run.vm06.stdout:Purging configuration files for ceph-mgr (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:49.462 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr, directory '/var/lib/ceph/mgr' not empty so not removed 2026-03-24T02:53:50.481 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:53:50.486 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:53:50.516 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:53:50.520 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:53:50.708 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:53:50.708 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:53:50.709 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:53:50.709 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:53:50.857 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:53:50.857 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:53:50.857 INFO:teuthology.orchestra.run.vm07.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:53:50.857 INFO:teuthology.orchestra.run.vm07.stdout: libsgutils2-2 libsqlite3-mod-ceph nvme-cli python-asyncssh-doc 2026-03-24T02:53:50.857 INFO:teuthology.orchestra.run.vm07.stdout: python3-asyncssh python3-cachetools python3-ceph-common python3-cheroot 2026-03-24T02:53:50.857 INFO:teuthology.orchestra.run.vm07.stdout: python3-cherrypy3 python3-google-auth python3-jaraco.classes 2026-03-24T02:53:50.857 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2026-03-24T02:53:50.858 INFO:teuthology.orchestra.run.vm07.stdout: python3-joblib python3-kubernetes python3-natsort python3-portend 2026-03-24T02:53:50.858 INFO:teuthology.orchestra.run.vm07.stdout: python3-prettytable python3-psutil python3-repoze.lru 2026-03-24T02:53:50.858 INFO:teuthology.orchestra.run.vm07.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-simplejson 2026-03-24T02:53:50.858 INFO:teuthology.orchestra.run.vm07.stdout: python3-sklearn python3-sklearn-lib python3-tempora python3-threadpoolctl 2026-03-24T02:53:50.858 INFO:teuthology.orchestra.run.vm07.stdout: python3-wcwidth python3-webob python3-websocket python3-zc.lockfile 2026-03-24T02:53:50.858 INFO:teuthology.orchestra.run.vm07.stdout: sg3-utils sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:53:50.858 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:53:50.872 INFO:teuthology.orchestra.run.vm07.stdout:The following packages will be REMOVED: 2026-03-24T02:53:50.874 INFO:teuthology.orchestra.run.vm07.stdout: ceph-base* ceph-common* ceph-mon* ceph-osd* ceph-test* ceph-volume* radosgw* 2026-03-24T02:53:50.874 INFO:teuthology.orchestra.run.vm07.stdout: rbd-mirror* 2026-03-24T02:53:50.889 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:53:50.889 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:53:50.889 INFO:teuthology.orchestra.run.vm06.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:53:50.890 INFO:teuthology.orchestra.run.vm06.stdout: libsgutils2-2 libsqlite3-mod-ceph nvme-cli python-asyncssh-doc 2026-03-24T02:53:50.890 INFO:teuthology.orchestra.run.vm06.stdout: python3-asyncssh python3-cachetools python3-ceph-common python3-cheroot 2026-03-24T02:53:50.890 INFO:teuthology.orchestra.run.vm06.stdout: python3-cherrypy3 python3-google-auth python3-jaraco.classes 2026-03-24T02:53:50.890 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2026-03-24T02:53:50.890 INFO:teuthology.orchestra.run.vm06.stdout: python3-joblib python3-kubernetes python3-natsort python3-portend 2026-03-24T02:53:50.890 INFO:teuthology.orchestra.run.vm06.stdout: python3-prettytable python3-psutil python3-repoze.lru 2026-03-24T02:53:50.890 INFO:teuthology.orchestra.run.vm06.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-simplejson 2026-03-24T02:53:50.890 INFO:teuthology.orchestra.run.vm06.stdout: python3-sklearn python3-sklearn-lib python3-tempora python3-threadpoolctl 2026-03-24T02:53:50.890 INFO:teuthology.orchestra.run.vm06.stdout: python3-wcwidth python3-webob python3-websocket python3-zc.lockfile 2026-03-24T02:53:50.890 INFO:teuthology.orchestra.run.vm06.stdout: sg3-utils sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:53:50.890 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:53:50.901 INFO:teuthology.orchestra.run.vm06.stdout:The following packages will be REMOVED: 2026-03-24T02:53:50.901 INFO:teuthology.orchestra.run.vm06.stdout: ceph-base* ceph-common* ceph-mon* ceph-osd* ceph-test* ceph-volume* radosgw* 2026-03-24T02:53:50.901 INFO:teuthology.orchestra.run.vm06.stdout: rbd-mirror* 2026-03-24T02:53:51.039 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 8 to remove and 44 not upgraded. 2026-03-24T02:53:51.039 INFO:teuthology.orchestra.run.vm07.stdout:After this operation, 743 MB disk space will be freed. 2026-03-24T02:53:51.064 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 8 to remove and 44 not upgraded. 2026-03-24T02:53:51.065 INFO:teuthology.orchestra.run.vm06.stdout:After this operation, 743 MB disk space will be freed. 2026-03-24T02:53:51.076 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 124278 files and directories currently installed.) 2026-03-24T02:53:51.077 INFO:teuthology.orchestra.run.vm07.stdout:Removing ceph-volume (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:51.101 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 124278 files and directories currently installed.) 2026-03-24T02:53:51.103 INFO:teuthology.orchestra.run.vm06.stdout:Removing ceph-volume (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:51.139 INFO:teuthology.orchestra.run.vm07.stdout:Removing ceph-osd (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:51.167 INFO:teuthology.orchestra.run.vm06.stdout:Removing ceph-osd (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:51.528 INFO:teuthology.orchestra.run.vm07.stdout:Removing ceph-mon (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:51.588 INFO:teuthology.orchestra.run.vm06.stdout:Removing ceph-mon (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:51.911 INFO:teuthology.orchestra.run.vm07.stdout:Removing ceph-base (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:52.010 INFO:teuthology.orchestra.run.vm06.stdout:Removing ceph-base (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:52.324 INFO:teuthology.orchestra.run.vm07.stdout:Removing rbd-mirror (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:52.436 INFO:teuthology.orchestra.run.vm06.stdout:Removing rbd-mirror (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:52.686 INFO:teuthology.orchestra.run.vm07.stdout:Removing radosgw (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:52.872 INFO:teuthology.orchestra.run.vm06.stdout:Removing radosgw (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:53.085 INFO:teuthology.orchestra.run.vm07.stdout:Removing ceph-test (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:53.109 INFO:teuthology.orchestra.run.vm07.stdout:Removing ceph-common (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:53.309 INFO:teuthology.orchestra.run.vm06.stdout:Removing ceph-test (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:53.333 INFO:teuthology.orchestra.run.vm06.stdout:Removing ceph-common (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:53.493 INFO:teuthology.orchestra.run.vm07.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-03-24T02:53:53.532 INFO:teuthology.orchestra.run.vm07.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-03-24T02:53:53.600 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123780 files and directories currently installed.) 2026-03-24T02:53:53.602 INFO:teuthology.orchestra.run.vm07.stdout:Purging configuration files for radosgw (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:53.758 INFO:teuthology.orchestra.run.vm06.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-03-24T02:53:53.792 INFO:teuthology.orchestra.run.vm06.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-03-24T02:53:53.862 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123780 files and directories currently installed.) 2026-03-24T02:53:53.865 INFO:teuthology.orchestra.run.vm06.stdout:Purging configuration files for radosgw (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:54.133 INFO:teuthology.orchestra.run.vm07.stdout:Purging configuration files for ceph-mon (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:54.431 INFO:teuthology.orchestra.run.vm06.stdout:Purging configuration files for ceph-mon (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:54.506 INFO:teuthology.orchestra.run.vm07.stdout:dpkg: warning: while removing ceph-mon, directory '/var/lib/ceph/mon' not empty so not removed 2026-03-24T02:53:54.516 INFO:teuthology.orchestra.run.vm07.stdout:Purging configuration files for ceph-base (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:54.859 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mon, directory '/var/lib/ceph/mon' not empty so not removed 2026-03-24T02:53:54.868 INFO:teuthology.orchestra.run.vm06.stdout:Purging configuration files for ceph-base (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:54.901 INFO:teuthology.orchestra.run.vm07.stdout:Purging configuration files for rbd-mirror (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:55.279 INFO:teuthology.orchestra.run.vm07.stdout:Purging configuration files for ceph-common (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:55.288 INFO:teuthology.orchestra.run.vm06.stdout:Purging configuration files for rbd-mirror (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:55.718 INFO:teuthology.orchestra.run.vm06.stdout:Purging configuration files for ceph-common (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:55.887 INFO:teuthology.orchestra.run.vm07.stdout:Purging configuration files for ceph-osd (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:56.247 INFO:teuthology.orchestra.run.vm07.stdout:dpkg: warning: while removing ceph-osd, directory '/var/lib/ceph/osd' not empty so not removed 2026-03-24T02:53:56.250 INFO:teuthology.orchestra.run.vm06.stdout:Purging configuration files for ceph-osd (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:56.637 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-osd, directory '/var/lib/ceph/osd' not empty so not removed 2026-03-24T02:53:57.310 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:53:57.345 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:53:57.505 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:53:57.506 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:53:57.634 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:53:57.634 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:53:57.634 INFO:teuthology.orchestra.run.vm07.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:53:57.635 INFO:teuthology.orchestra.run.vm07.stdout: libsgutils2-2 libsqlite3-mod-ceph nvme-cli python-asyncssh-doc 2026-03-24T02:53:57.635 INFO:teuthology.orchestra.run.vm07.stdout: python3-asyncssh python3-cachetools python3-ceph-common python3-cheroot 2026-03-24T02:53:57.635 INFO:teuthology.orchestra.run.vm07.stdout: python3-cherrypy3 python3-google-auth python3-jaraco.classes 2026-03-24T02:53:57.635 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2026-03-24T02:53:57.635 INFO:teuthology.orchestra.run.vm07.stdout: python3-joblib python3-kubernetes python3-natsort python3-portend 2026-03-24T02:53:57.635 INFO:teuthology.orchestra.run.vm07.stdout: python3-prettytable python3-psutil python3-repoze.lru 2026-03-24T02:53:57.635 INFO:teuthology.orchestra.run.vm07.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-simplejson 2026-03-24T02:53:57.635 INFO:teuthology.orchestra.run.vm07.stdout: python3-sklearn python3-sklearn-lib python3-tempora python3-threadpoolctl 2026-03-24T02:53:57.635 INFO:teuthology.orchestra.run.vm07.stdout: python3-wcwidth python3-webob python3-websocket python3-zc.lockfile 2026-03-24T02:53:57.635 INFO:teuthology.orchestra.run.vm07.stdout: sg3-utils sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:53:57.635 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:53:57.648 INFO:teuthology.orchestra.run.vm07.stdout:The following packages will be REMOVED: 2026-03-24T02:53:57.649 INFO:teuthology.orchestra.run.vm07.stdout: ceph-fuse* 2026-03-24T02:53:57.674 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:53:57.712 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:53:57.817 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 1 to remove and 44 not upgraded. 2026-03-24T02:53:57.817 INFO:teuthology.orchestra.run.vm07.stdout:After this operation, 2932 kB disk space will be freed. 2026-03-24T02:53:57.855 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123764 files and directories currently installed.) 2026-03-24T02:53:57.857 INFO:teuthology.orchestra.run.vm07.stdout:Removing ceph-fuse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:57.900 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:53:57.900 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:53:58.025 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:53:58.025 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:53:58.025 INFO:teuthology.orchestra.run.vm06.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:53:58.026 INFO:teuthology.orchestra.run.vm06.stdout: libsgutils2-2 libsqlite3-mod-ceph nvme-cli python-asyncssh-doc 2026-03-24T02:53:58.026 INFO:teuthology.orchestra.run.vm06.stdout: python3-asyncssh python3-cachetools python3-ceph-common python3-cheroot 2026-03-24T02:53:58.026 INFO:teuthology.orchestra.run.vm06.stdout: python3-cherrypy3 python3-google-auth python3-jaraco.classes 2026-03-24T02:53:58.026 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2026-03-24T02:53:58.026 INFO:teuthology.orchestra.run.vm06.stdout: python3-joblib python3-kubernetes python3-natsort python3-portend 2026-03-24T02:53:58.026 INFO:teuthology.orchestra.run.vm06.stdout: python3-prettytable python3-psutil python3-repoze.lru 2026-03-24T02:53:58.026 INFO:teuthology.orchestra.run.vm06.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-simplejson 2026-03-24T02:53:58.026 INFO:teuthology.orchestra.run.vm06.stdout: python3-sklearn python3-sklearn-lib python3-tempora python3-threadpoolctl 2026-03-24T02:53:58.026 INFO:teuthology.orchestra.run.vm06.stdout: python3-wcwidth python3-webob python3-websocket python3-zc.lockfile 2026-03-24T02:53:58.026 INFO:teuthology.orchestra.run.vm06.stdout: sg3-utils sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:53:58.026 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:53:58.037 INFO:teuthology.orchestra.run.vm06.stdout:The following packages will be REMOVED: 2026-03-24T02:53:58.037 INFO:teuthology.orchestra.run.vm06.stdout: ceph-fuse* 2026-03-24T02:53:58.204 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 1 to remove and 44 not upgraded. 2026-03-24T02:53:58.204 INFO:teuthology.orchestra.run.vm06.stdout:After this operation, 2932 kB disk space will be freed. 2026-03-24T02:53:58.231 INFO:teuthology.orchestra.run.vm07.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-03-24T02:53:58.241 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123764 files and directories currently installed.) 2026-03-24T02:53:58.243 INFO:teuthology.orchestra.run.vm06.stdout:Removing ceph-fuse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:58.319 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123755 files and directories currently installed.) 2026-03-24T02:53:58.321 INFO:teuthology.orchestra.run.vm07.stdout:Purging configuration files for ceph-fuse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:58.685 INFO:teuthology.orchestra.run.vm06.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-03-24T02:53:58.781 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123755 files and directories currently installed.) 2026-03-24T02:53:58.784 INFO:teuthology.orchestra.run.vm06.stdout:Purging configuration files for ceph-fuse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:53:59.684 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:53:59.720 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:53:59.895 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:53:59.896 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:54:00.017 INFO:teuthology.orchestra.run.vm07.stdout:Package 'ceph-test' is not installed, so not removed 2026-03-24T02:54:00.045 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:00.045 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:00.045 INFO:teuthology.orchestra.run.vm07.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:54:00.045 INFO:teuthology.orchestra.run.vm07.stdout: libsgutils2-2 libsqlite3-mod-ceph nvme-cli python-asyncssh-doc 2026-03-24T02:54:00.045 INFO:teuthology.orchestra.run.vm07.stdout: python3-asyncssh python3-cachetools python3-ceph-common python3-cheroot 2026-03-24T02:54:00.045 INFO:teuthology.orchestra.run.vm07.stdout: python3-cherrypy3 python3-google-auth python3-jaraco.classes 2026-03-24T02:54:00.045 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2026-03-24T02:54:00.045 INFO:teuthology.orchestra.run.vm07.stdout: python3-joblib python3-kubernetes python3-natsort python3-portend 2026-03-24T02:54:00.045 INFO:teuthology.orchestra.run.vm07.stdout: python3-prettytable python3-psutil python3-repoze.lru 2026-03-24T02:54:00.045 INFO:teuthology.orchestra.run.vm07.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-simplejson 2026-03-24T02:54:00.045 INFO:teuthology.orchestra.run.vm07.stdout: python3-sklearn python3-sklearn-lib python3-tempora python3-threadpoolctl 2026-03-24T02:54:00.045 INFO:teuthology.orchestra.run.vm07.stdout: python3-wcwidth python3-webob python3-websocket python3-zc.lockfile 2026-03-24T02:54:00.045 INFO:teuthology.orchestra.run.vm07.stdout: sg3-utils sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:54:00.045 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:00.046 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:00.046 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:00.068 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:54:00.236 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:54:00.237 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:54:00.272 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:00.311 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:54:00.358 INFO:teuthology.orchestra.run.vm07.stdout:Package 'ceph-volume' is not installed, so not removed 2026-03-24T02:54:00.358 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:00.358 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:00.358 INFO:teuthology.orchestra.run.vm07.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:54:00.358 INFO:teuthology.orchestra.run.vm07.stdout: libsgutils2-2 libsqlite3-mod-ceph nvme-cli python-asyncssh-doc 2026-03-24T02:54:00.358 INFO:teuthology.orchestra.run.vm07.stdout: python3-asyncssh python3-cachetools python3-ceph-common python3-cheroot 2026-03-24T02:54:00.358 INFO:teuthology.orchestra.run.vm07.stdout: python3-cherrypy3 python3-google-auth python3-jaraco.classes 2026-03-24T02:54:00.358 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2026-03-24T02:54:00.358 INFO:teuthology.orchestra.run.vm07.stdout: python3-joblib python3-kubernetes python3-natsort python3-portend 2026-03-24T02:54:00.358 INFO:teuthology.orchestra.run.vm07.stdout: python3-prettytable python3-psutil python3-repoze.lru 2026-03-24T02:54:00.358 INFO:teuthology.orchestra.run.vm07.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-simplejson 2026-03-24T02:54:00.358 INFO:teuthology.orchestra.run.vm07.stdout: python3-sklearn python3-sklearn-lib python3-tempora python3-threadpoolctl 2026-03-24T02:54:00.358 INFO:teuthology.orchestra.run.vm07.stdout: python3-wcwidth python3-webob python3-websocket python3-zc.lockfile 2026-03-24T02:54:00.358 INFO:teuthology.orchestra.run.vm07.stdout: sg3-utils sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:54:00.358 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:00.377 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:00.377 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:00.412 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:54:00.478 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:54:00.478 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:54:00.612 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:54:00.613 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:54:00.621 INFO:teuthology.orchestra.run.vm06.stdout:Package 'ceph-test' is not installed, so not removed 2026-03-24T02:54:00.621 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:00.621 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:00.621 INFO:teuthology.orchestra.run.vm06.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:54:00.621 INFO:teuthology.orchestra.run.vm06.stdout: libsgutils2-2 libsqlite3-mod-ceph nvme-cli python-asyncssh-doc 2026-03-24T02:54:00.621 INFO:teuthology.orchestra.run.vm06.stdout: python3-asyncssh python3-cachetools python3-ceph-common python3-cheroot 2026-03-24T02:54:00.621 INFO:teuthology.orchestra.run.vm06.stdout: python3-cherrypy3 python3-google-auth python3-jaraco.classes 2026-03-24T02:54:00.621 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2026-03-24T02:54:00.621 INFO:teuthology.orchestra.run.vm06.stdout: python3-joblib python3-kubernetes python3-natsort python3-portend 2026-03-24T02:54:00.621 INFO:teuthology.orchestra.run.vm06.stdout: python3-prettytable python3-psutil python3-repoze.lru 2026-03-24T02:54:00.621 INFO:teuthology.orchestra.run.vm06.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-simplejson 2026-03-24T02:54:00.621 INFO:teuthology.orchestra.run.vm06.stdout: python3-sklearn python3-sklearn-lib python3-tempora python3-threadpoolctl 2026-03-24T02:54:00.621 INFO:teuthology.orchestra.run.vm06.stdout: python3-wcwidth python3-webob python3-websocket python3-zc.lockfile 2026-03-24T02:54:00.621 INFO:teuthology.orchestra.run.vm06.stdout: sg3-utils sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:54:00.621 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:00.646 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:00.646 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:00.685 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:54:00.728 INFO:teuthology.orchestra.run.vm07.stdout:Package 'radosgw' is not installed, so not removed 2026-03-24T02:54:00.729 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:00.729 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:00.729 INFO:teuthology.orchestra.run.vm07.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:54:00.729 INFO:teuthology.orchestra.run.vm07.stdout: libsgutils2-2 libsqlite3-mod-ceph nvme-cli python-asyncssh-doc 2026-03-24T02:54:00.729 INFO:teuthology.orchestra.run.vm07.stdout: python3-asyncssh python3-cachetools python3-ceph-common python3-cheroot 2026-03-24T02:54:00.729 INFO:teuthology.orchestra.run.vm07.stdout: python3-cherrypy3 python3-google-auth python3-jaraco.classes 2026-03-24T02:54:00.729 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2026-03-24T02:54:00.729 INFO:teuthology.orchestra.run.vm07.stdout: python3-joblib python3-kubernetes python3-natsort python3-portend 2026-03-24T02:54:00.729 INFO:teuthology.orchestra.run.vm07.stdout: python3-prettytable python3-psutil python3-repoze.lru 2026-03-24T02:54:00.729 INFO:teuthology.orchestra.run.vm07.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-simplejson 2026-03-24T02:54:00.729 INFO:teuthology.orchestra.run.vm07.stdout: python3-sklearn python3-sklearn-lib python3-tempora python3-threadpoolctl 2026-03-24T02:54:00.729 INFO:teuthology.orchestra.run.vm07.stdout: python3-wcwidth python3-webob python3-websocket python3-zc.lockfile 2026-03-24T02:54:00.729 INFO:teuthology.orchestra.run.vm07.stdout: sg3-utils sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:54:00.729 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:00.746 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:00.746 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:00.780 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:54:00.859 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:54:00.859 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:54:00.953 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:54:00.954 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:54:00.972 INFO:teuthology.orchestra.run.vm06.stdout:Package 'ceph-volume' is not installed, so not removed 2026-03-24T02:54:00.972 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:00.972 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:00.972 INFO:teuthology.orchestra.run.vm06.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:54:00.972 INFO:teuthology.orchestra.run.vm06.stdout: libsgutils2-2 libsqlite3-mod-ceph nvme-cli python-asyncssh-doc 2026-03-24T02:54:00.972 INFO:teuthology.orchestra.run.vm06.stdout: python3-asyncssh python3-cachetools python3-ceph-common python3-cheroot 2026-03-24T02:54:00.972 INFO:teuthology.orchestra.run.vm06.stdout: python3-cherrypy3 python3-google-auth python3-jaraco.classes 2026-03-24T02:54:00.972 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2026-03-24T02:54:00.972 INFO:teuthology.orchestra.run.vm06.stdout: python3-joblib python3-kubernetes python3-natsort python3-portend 2026-03-24T02:54:00.972 INFO:teuthology.orchestra.run.vm06.stdout: python3-prettytable python3-psutil python3-repoze.lru 2026-03-24T02:54:00.972 INFO:teuthology.orchestra.run.vm06.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-simplejson 2026-03-24T02:54:00.972 INFO:teuthology.orchestra.run.vm06.stdout: python3-sklearn python3-sklearn-lib python3-tempora python3-threadpoolctl 2026-03-24T02:54:00.972 INFO:teuthology.orchestra.run.vm06.stdout: python3-wcwidth python3-webob python3-websocket python3-zc.lockfile 2026-03-24T02:54:00.972 INFO:teuthology.orchestra.run.vm06.stdout: sg3-utils sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:54:00.972 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:00.991 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:00.992 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:01.025 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:54:01.080 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:01.080 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:01.080 INFO:teuthology.orchestra.run.vm07.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:54:01.080 INFO:teuthology.orchestra.run.vm07.stdout: librdkafka1 librgw2 libsgutils2-2 libsqlite3-mod-ceph nvme-cli 2026-03-24T02:54:01.080 INFO:teuthology.orchestra.run.vm07.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:01.081 INFO:teuthology.orchestra.run.vm07.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:01.081 INFO:teuthology.orchestra.run.vm07.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:01.081 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:01.081 INFO:teuthology.orchestra.run.vm07.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:01.081 INFO:teuthology.orchestra.run.vm07.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:01.081 INFO:teuthology.orchestra.run.vm07.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:01.081 INFO:teuthology.orchestra.run.vm07.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:01.081 INFO:teuthology.orchestra.run.vm07.stdout: python3-websocket python3-zc.lockfile sg3-utils sg3-utils-udev smartmontools 2026-03-24T02:54:01.081 INFO:teuthology.orchestra.run.vm07.stdout: socat xmlstarlet 2026-03-24T02:54:01.081 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:01.089 INFO:teuthology.orchestra.run.vm07.stdout:The following packages will be REMOVED: 2026-03-24T02:54:01.090 INFO:teuthology.orchestra.run.vm07.stdout: python3-cephfs* python3-rados* python3-rgw* 2026-03-24T02:54:01.198 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:54:01.198 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:54:01.252 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 3 to remove and 44 not upgraded. 2026-03-24T02:54:01.252 INFO:teuthology.orchestra.run.vm07.stdout:After this operation, 2086 kB disk space will be freed. 2026-03-24T02:54:01.290 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123755 files and directories currently installed.) 2026-03-24T02:54:01.292 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-cephfs (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:01.304 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-rgw (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:01.310 INFO:teuthology.orchestra.run.vm06.stdout:Package 'radosgw' is not installed, so not removed 2026-03-24T02:54:01.310 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:01.310 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:01.310 INFO:teuthology.orchestra.run.vm06.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:54:01.311 INFO:teuthology.orchestra.run.vm06.stdout: libsgutils2-2 libsqlite3-mod-ceph nvme-cli python-asyncssh-doc 2026-03-24T02:54:01.311 INFO:teuthology.orchestra.run.vm06.stdout: python3-asyncssh python3-cachetools python3-ceph-common python3-cheroot 2026-03-24T02:54:01.311 INFO:teuthology.orchestra.run.vm06.stdout: python3-cherrypy3 python3-google-auth python3-jaraco.classes 2026-03-24T02:54:01.311 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2026-03-24T02:54:01.311 INFO:teuthology.orchestra.run.vm06.stdout: python3-joblib python3-kubernetes python3-natsort python3-portend 2026-03-24T02:54:01.311 INFO:teuthology.orchestra.run.vm06.stdout: python3-prettytable python3-psutil python3-repoze.lru 2026-03-24T02:54:01.311 INFO:teuthology.orchestra.run.vm06.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-simplejson 2026-03-24T02:54:01.311 INFO:teuthology.orchestra.run.vm06.stdout: python3-sklearn python3-sklearn-lib python3-tempora python3-threadpoolctl 2026-03-24T02:54:01.311 INFO:teuthology.orchestra.run.vm06.stdout: python3-wcwidth python3-webob python3-websocket python3-zc.lockfile 2026-03-24T02:54:01.311 INFO:teuthology.orchestra.run.vm06.stdout: sg3-utils sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:54:01.311 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:01.315 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-rados (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:01.328 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:01.328 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:01.363 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:54:01.542 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:54:01.542 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:54:01.664 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:01.664 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:01.664 INFO:teuthology.orchestra.run.vm06.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:54:01.664 INFO:teuthology.orchestra.run.vm06.stdout: librdkafka1 librgw2 libsgutils2-2 libsqlite3-mod-ceph nvme-cli 2026-03-24T02:54:01.664 INFO:teuthology.orchestra.run.vm06.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:01.664 INFO:teuthology.orchestra.run.vm06.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:01.664 INFO:teuthology.orchestra.run.vm06.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:01.664 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:01.664 INFO:teuthology.orchestra.run.vm06.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:01.664 INFO:teuthology.orchestra.run.vm06.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:01.664 INFO:teuthology.orchestra.run.vm06.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:01.664 INFO:teuthology.orchestra.run.vm06.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:01.664 INFO:teuthology.orchestra.run.vm06.stdout: python3-websocket python3-zc.lockfile sg3-utils sg3-utils-udev smartmontools 2026-03-24T02:54:01.664 INFO:teuthology.orchestra.run.vm06.stdout: socat xmlstarlet 2026-03-24T02:54:01.664 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:01.675 INFO:teuthology.orchestra.run.vm06.stdout:The following packages will be REMOVED: 2026-03-24T02:54:01.676 INFO:teuthology.orchestra.run.vm06.stdout: python3-cephfs* python3-rados* python3-rgw* 2026-03-24T02:54:01.834 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 3 to remove and 44 not upgraded. 2026-03-24T02:54:01.834 INFO:teuthology.orchestra.run.vm06.stdout:After this operation, 2086 kB disk space will be freed. 2026-03-24T02:54:01.870 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123755 files and directories currently installed.) 2026-03-24T02:54:01.872 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-cephfs (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:01.884 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-rgw (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:01.895 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-rados (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:02.363 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:02.404 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:54:02.578 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:54:02.579 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:54:02.687 INFO:teuthology.orchestra.run.vm07.stdout:Package 'python3-rgw' is not installed, so not removed 2026-03-24T02:54:02.687 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:02.687 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:02.687 INFO:teuthology.orchestra.run.vm07.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:54:02.687 INFO:teuthology.orchestra.run.vm07.stdout: librdkafka1 librgw2 libsgutils2-2 libsqlite3-mod-ceph nvme-cli 2026-03-24T02:54:02.687 INFO:teuthology.orchestra.run.vm07.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:02.687 INFO:teuthology.orchestra.run.vm07.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:02.687 INFO:teuthology.orchestra.run.vm07.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:02.687 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:02.687 INFO:teuthology.orchestra.run.vm07.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:02.687 INFO:teuthology.orchestra.run.vm07.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:02.687 INFO:teuthology.orchestra.run.vm07.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:02.688 INFO:teuthology.orchestra.run.vm07.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:02.688 INFO:teuthology.orchestra.run.vm07.stdout: python3-websocket python3-zc.lockfile sg3-utils sg3-utils-udev smartmontools 2026-03-24T02:54:02.688 INFO:teuthology.orchestra.run.vm07.stdout: socat xmlstarlet 2026-03-24T02:54:02.688 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:02.704 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:02.704 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:02.738 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:54:02.894 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:54:02.894 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:54:02.905 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:02.941 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:54:03.009 INFO:teuthology.orchestra.run.vm07.stdout:Package 'python3-cephfs' is not installed, so not removed 2026-03-24T02:54:03.009 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:03.009 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:03.009 INFO:teuthology.orchestra.run.vm07.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:54:03.009 INFO:teuthology.orchestra.run.vm07.stdout: librdkafka1 librgw2 libsgutils2-2 libsqlite3-mod-ceph nvme-cli 2026-03-24T02:54:03.009 INFO:teuthology.orchestra.run.vm07.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:03.009 INFO:teuthology.orchestra.run.vm07.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:03.009 INFO:teuthology.orchestra.run.vm07.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:03.009 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:03.009 INFO:teuthology.orchestra.run.vm07.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:03.009 INFO:teuthology.orchestra.run.vm07.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:03.009 INFO:teuthology.orchestra.run.vm07.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:03.009 INFO:teuthology.orchestra.run.vm07.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:03.009 INFO:teuthology.orchestra.run.vm07.stdout: python3-websocket python3-zc.lockfile sg3-utils sg3-utils-udev smartmontools 2026-03-24T02:54:03.009 INFO:teuthology.orchestra.run.vm07.stdout: socat xmlstarlet 2026-03-24T02:54:03.009 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:03.025 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:03.026 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:03.060 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:54:03.110 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:54:03.110 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:54:03.215 INFO:teuthology.orchestra.run.vm06.stdout:Package 'python3-rgw' is not installed, so not removed 2026-03-24T02:54:03.215 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:03.215 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:03.215 INFO:teuthology.orchestra.run.vm06.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:54:03.215 INFO:teuthology.orchestra.run.vm06.stdout: librdkafka1 librgw2 libsgutils2-2 libsqlite3-mod-ceph nvme-cli 2026-03-24T02:54:03.215 INFO:teuthology.orchestra.run.vm06.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:03.216 INFO:teuthology.orchestra.run.vm06.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:03.216 INFO:teuthology.orchestra.run.vm06.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:03.216 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:03.216 INFO:teuthology.orchestra.run.vm06.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:03.216 INFO:teuthology.orchestra.run.vm06.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:03.216 INFO:teuthology.orchestra.run.vm06.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:03.216 INFO:teuthology.orchestra.run.vm06.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:03.216 INFO:teuthology.orchestra.run.vm06.stdout: python3-websocket python3-zc.lockfile sg3-utils sg3-utils-udev smartmontools 2026-03-24T02:54:03.216 INFO:teuthology.orchestra.run.vm06.stdout: socat xmlstarlet 2026-03-24T02:54:03.216 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:03.232 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:03.232 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:03.232 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:54:03.233 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:54:03.265 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:54:03.356 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:03.356 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:03.356 INFO:teuthology.orchestra.run.vm07.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:54:03.356 INFO:teuthology.orchestra.run.vm07.stdout: librdkafka1 librgw2 libsgutils2-2 libsqlite3-mod-ceph nvme-cli 2026-03-24T02:54:03.356 INFO:teuthology.orchestra.run.vm07.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:03.356 INFO:teuthology.orchestra.run.vm07.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:03.356 INFO:teuthology.orchestra.run.vm07.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:03.356 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:03.356 INFO:teuthology.orchestra.run.vm07.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:03.356 INFO:teuthology.orchestra.run.vm07.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:03.356 INFO:teuthology.orchestra.run.vm07.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:03.356 INFO:teuthology.orchestra.run.vm07.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:03.356 INFO:teuthology.orchestra.run.vm07.stdout: python3-websocket python3-zc.lockfile sg3-utils sg3-utils-udev smartmontools 2026-03-24T02:54:03.356 INFO:teuthology.orchestra.run.vm07.stdout: socat xmlstarlet 2026-03-24T02:54:03.356 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:03.364 INFO:teuthology.orchestra.run.vm07.stdout:The following packages will be REMOVED: 2026-03-24T02:54:03.364 INFO:teuthology.orchestra.run.vm07.stdout: python3-rbd* 2026-03-24T02:54:03.427 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:54:03.427 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:54:03.524 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 1 to remove and 44 not upgraded. 2026-03-24T02:54:03.524 INFO:teuthology.orchestra.run.vm07.stdout:After this operation, 1205 kB disk space will be freed. 2026-03-24T02:54:03.537 INFO:teuthology.orchestra.run.vm06.stdout:Package 'python3-cephfs' is not installed, so not removed 2026-03-24T02:54:03.537 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:03.537 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:03.537 INFO:teuthology.orchestra.run.vm06.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:54:03.537 INFO:teuthology.orchestra.run.vm06.stdout: librdkafka1 librgw2 libsgutils2-2 libsqlite3-mod-ceph nvme-cli 2026-03-24T02:54:03.537 INFO:teuthology.orchestra.run.vm06.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:03.537 INFO:teuthology.orchestra.run.vm06.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:03.537 INFO:teuthology.orchestra.run.vm06.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:03.537 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:03.537 INFO:teuthology.orchestra.run.vm06.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:03.537 INFO:teuthology.orchestra.run.vm06.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:03.537 INFO:teuthology.orchestra.run.vm06.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:03.537 INFO:teuthology.orchestra.run.vm06.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:03.537 INFO:teuthology.orchestra.run.vm06.stdout: python3-websocket python3-zc.lockfile sg3-utils sg3-utils-udev smartmontools 2026-03-24T02:54:03.537 INFO:teuthology.orchestra.run.vm06.stdout: socat xmlstarlet 2026-03-24T02:54:03.537 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:03.555 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:03.556 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:03.564 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123731 files and directories currently installed.) 2026-03-24T02:54:03.566 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-rbd (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:03.591 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:54:03.777 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:54:03.778 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:54:03.888 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:03.888 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:03.888 INFO:teuthology.orchestra.run.vm06.stdout: libboost-thread1.74.0 libjq1 liboath0 libonig5 libpmemobj1 libradosstriper1 2026-03-24T02:54:03.888 INFO:teuthology.orchestra.run.vm06.stdout: librdkafka1 librgw2 libsgutils2-2 libsqlite3-mod-ceph nvme-cli 2026-03-24T02:54:03.888 INFO:teuthology.orchestra.run.vm06.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:03.888 INFO:teuthology.orchestra.run.vm06.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:03.888 INFO:teuthology.orchestra.run.vm06.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:03.888 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:03.888 INFO:teuthology.orchestra.run.vm06.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:03.888 INFO:teuthology.orchestra.run.vm06.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:03.888 INFO:teuthology.orchestra.run.vm06.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:03.888 INFO:teuthology.orchestra.run.vm06.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:03.888 INFO:teuthology.orchestra.run.vm06.stdout: python3-websocket python3-zc.lockfile sg3-utils sg3-utils-udev smartmontools 2026-03-24T02:54:03.888 INFO:teuthology.orchestra.run.vm06.stdout: socat xmlstarlet 2026-03-24T02:54:03.888 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:03.897 INFO:teuthology.orchestra.run.vm06.stdout:The following packages will be REMOVED: 2026-03-24T02:54:03.897 INFO:teuthology.orchestra.run.vm06.stdout: python3-rbd* 2026-03-24T02:54:04.059 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 1 to remove and 44 not upgraded. 2026-03-24T02:54:04.059 INFO:teuthology.orchestra.run.vm06.stdout:After this operation, 1205 kB disk space will be freed. 2026-03-24T02:54:04.101 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123731 files and directories currently installed.) 2026-03-24T02:54:04.103 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-rbd (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:04.660 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:04.696 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:54:04.862 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:54:04.863 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:54:04.974 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:04.974 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:04.974 INFO:teuthology.orchestra.run.vm07.stdout: libboost-thread1.74.0 libcephfs-proxy2 libjq1 liboath0 libonig5 libpmemobj1 2026-03-24T02:54:04.974 INFO:teuthology.orchestra.run.vm07.stdout: libradosstriper1 librdkafka1 librgw2 libsgutils2-2 libsqlite3-mod-ceph 2026-03-24T02:54:04.974 INFO:teuthology.orchestra.run.vm07.stdout: nvme-cli python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:04.974 INFO:teuthology.orchestra.run.vm07.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:04.974 INFO:teuthology.orchestra.run.vm07.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:04.974 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:04.974 INFO:teuthology.orchestra.run.vm07.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:04.974 INFO:teuthology.orchestra.run.vm07.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:04.974 INFO:teuthology.orchestra.run.vm07.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:04.974 INFO:teuthology.orchestra.run.vm07.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:04.974 INFO:teuthology.orchestra.run.vm07.stdout: python3-websocket python3-zc.lockfile sg3-utils sg3-utils-udev smartmontools 2026-03-24T02:54:04.974 INFO:teuthology.orchestra.run.vm07.stdout: socat xmlstarlet 2026-03-24T02:54:04.974 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:04.983 INFO:teuthology.orchestra.run.vm07.stdout:The following packages will be REMOVED: 2026-03-24T02:54:04.983 INFO:teuthology.orchestra.run.vm07.stdout: libcephfs-daemon* libcephfs-dev* libcephfs2* 2026-03-24T02:54:05.117 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:05.136 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 3 to remove and 44 not upgraded. 2026-03-24T02:54:05.136 INFO:teuthology.orchestra.run.vm07.stdout:After this operation, 2851 kB disk space will be freed. 2026-03-24T02:54:05.152 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:54:05.173 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123723 files and directories currently installed.) 2026-03-24T02:54:05.175 INFO:teuthology.orchestra.run.vm07.stdout:Removing libcephfs-daemon (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:05.186 INFO:teuthology.orchestra.run.vm07.stdout:Removing libcephfs-dev (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:05.197 INFO:teuthology.orchestra.run.vm07.stdout:Removing libcephfs2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:05.221 INFO:teuthology.orchestra.run.vm07.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-03-24T02:54:05.329 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:54:05.329 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:54:05.460 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:05.460 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:05.460 INFO:teuthology.orchestra.run.vm06.stdout: libboost-thread1.74.0 libcephfs-proxy2 libjq1 liboath0 libonig5 libpmemobj1 2026-03-24T02:54:05.460 INFO:teuthology.orchestra.run.vm06.stdout: libradosstriper1 librdkafka1 librgw2 libsgutils2-2 libsqlite3-mod-ceph 2026-03-24T02:54:05.460 INFO:teuthology.orchestra.run.vm06.stdout: nvme-cli python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:05.460 INFO:teuthology.orchestra.run.vm06.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:05.460 INFO:teuthology.orchestra.run.vm06.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:05.460 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:05.460 INFO:teuthology.orchestra.run.vm06.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:05.460 INFO:teuthology.orchestra.run.vm06.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:05.460 INFO:teuthology.orchestra.run.vm06.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:05.460 INFO:teuthology.orchestra.run.vm06.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:05.460 INFO:teuthology.orchestra.run.vm06.stdout: python3-websocket python3-zc.lockfile sg3-utils sg3-utils-udev smartmontools 2026-03-24T02:54:05.460 INFO:teuthology.orchestra.run.vm06.stdout: socat xmlstarlet 2026-03-24T02:54:05.460 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:05.471 INFO:teuthology.orchestra.run.vm06.stdout:The following packages will be REMOVED: 2026-03-24T02:54:05.471 INFO:teuthology.orchestra.run.vm06.stdout: libcephfs-daemon* libcephfs-dev* libcephfs2* 2026-03-24T02:54:05.765 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 3 to remove and 44 not upgraded. 2026-03-24T02:54:05.765 INFO:teuthology.orchestra.run.vm06.stdout:After this operation, 2851 kB disk space will be freed. 2026-03-24T02:54:05.804 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123723 files and directories currently installed.) 2026-03-24T02:54:05.806 INFO:teuthology.orchestra.run.vm06.stdout:Removing libcephfs-daemon (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:05.817 INFO:teuthology.orchestra.run.vm06.stdout:Removing libcephfs-dev (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:05.828 INFO:teuthology.orchestra.run.vm06.stdout:Removing libcephfs2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:05.851 INFO:teuthology.orchestra.run.vm06.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-03-24T02:54:06.226 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:06.261 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:54:06.441 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:54:06.441 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:54:06.559 INFO:teuthology.orchestra.run.vm07.stdout:Package 'libcephfs-dev' is not installed, so not removed 2026-03-24T02:54:06.559 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:06.559 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:06.559 INFO:teuthology.orchestra.run.vm07.stdout: libboost-thread1.74.0 libcephfs-proxy2 libjq1 liboath0 libonig5 libpmemobj1 2026-03-24T02:54:06.559 INFO:teuthology.orchestra.run.vm07.stdout: libradosstriper1 librdkafka1 librgw2 libsgutils2-2 libsqlite3-mod-ceph 2026-03-24T02:54:06.559 INFO:teuthology.orchestra.run.vm07.stdout: nvme-cli python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:06.559 INFO:teuthology.orchestra.run.vm07.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:06.559 INFO:teuthology.orchestra.run.vm07.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:06.560 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:06.560 INFO:teuthology.orchestra.run.vm07.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:06.560 INFO:teuthology.orchestra.run.vm07.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:06.560 INFO:teuthology.orchestra.run.vm07.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:06.560 INFO:teuthology.orchestra.run.vm07.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:06.560 INFO:teuthology.orchestra.run.vm07.stdout: python3-websocket python3-zc.lockfile sg3-utils sg3-utils-udev smartmontools 2026-03-24T02:54:06.560 INFO:teuthology.orchestra.run.vm07.stdout: socat xmlstarlet 2026-03-24T02:54:06.560 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:06.580 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:06.580 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:06.617 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:54:06.796 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:54:06.796 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:54:06.889 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:06.915 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:06.915 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:06.915 INFO:teuthology.orchestra.run.vm07.stdout: libboost-thread1.74.0 libcephfs-proxy2 libdouble-conversion3 libfuse2 2026-03-24T02:54:06.915 INFO:teuthology.orchestra.run.vm07.stdout: libgfapi0 libgfrpc0 libgfxdr0 libglusterfs0 libiscsi7 libjq1 liblttng-ust1 2026-03-24T02:54:06.915 INFO:teuthology.orchestra.run.vm07.stdout: libnbd0 liboath0 libonig5 libpcre2-16-0 libpmemobj1 libqt5core5a libqt5dbus5 2026-03-24T02:54:06.915 INFO:teuthology.orchestra.run.vm07.stdout: libqt5network5 librdkafka1 libsgutils2-2 libthrift-0.16.0 nvme-cli 2026-03-24T02:54:06.915 INFO:teuthology.orchestra.run.vm07.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:06.915 INFO:teuthology.orchestra.run.vm07.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:06.915 INFO:teuthology.orchestra.run.vm07.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:06.915 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:06.915 INFO:teuthology.orchestra.run.vm07.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:06.915 INFO:teuthology.orchestra.run.vm07.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:06.915 INFO:teuthology.orchestra.run.vm07.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:06.916 INFO:teuthology.orchestra.run.vm07.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:06.916 INFO:teuthology.orchestra.run.vm07.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n sg3-utils 2026-03-24T02:54:06.916 INFO:teuthology.orchestra.run.vm07.stdout: sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:54:06.916 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:06.924 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:54:06.928 INFO:teuthology.orchestra.run.vm07.stdout:The following packages will be REMOVED: 2026-03-24T02:54:06.928 INFO:teuthology.orchestra.run.vm07.stdout: librados2* libradosstriper1* librbd1* librgw2* libsqlite3-mod-ceph* 2026-03-24T02:54:06.928 INFO:teuthology.orchestra.run.vm07.stdout: qemu-block-extra* rbd-fuse* 2026-03-24T02:54:07.077 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:54:07.077 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:54:07.086 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 7 to remove and 44 not upgraded. 2026-03-24T02:54:07.086 INFO:teuthology.orchestra.run.vm07.stdout:After this operation, 59.2 MB disk space will be freed. 2026-03-24T02:54:07.127 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123701 files and directories currently installed.) 2026-03-24T02:54:07.130 INFO:teuthology.orchestra.run.vm07.stdout:Removing rbd-fuse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:07.142 INFO:teuthology.orchestra.run.vm07.stdout:Removing libsqlite3-mod-ceph (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:07.154 INFO:teuthology.orchestra.run.vm07.stdout:Removing libradosstriper1 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:07.164 INFO:teuthology.orchestra.run.vm07.stdout:Removing qemu-block-extra (1:6.2+dfsg-2ubuntu6.28) ... 2026-03-24T02:54:07.191 INFO:teuthology.orchestra.run.vm06.stdout:Package 'libcephfs-dev' is not installed, so not removed 2026-03-24T02:54:07.191 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:07.191 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:07.191 INFO:teuthology.orchestra.run.vm06.stdout: libboost-thread1.74.0 libcephfs-proxy2 libjq1 liboath0 libonig5 libpmemobj1 2026-03-24T02:54:07.191 INFO:teuthology.orchestra.run.vm06.stdout: libradosstriper1 librdkafka1 librgw2 libsgutils2-2 libsqlite3-mod-ceph 2026-03-24T02:54:07.192 INFO:teuthology.orchestra.run.vm06.stdout: nvme-cli python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:07.192 INFO:teuthology.orchestra.run.vm06.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:07.192 INFO:teuthology.orchestra.run.vm06.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:07.192 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:07.192 INFO:teuthology.orchestra.run.vm06.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:07.192 INFO:teuthology.orchestra.run.vm06.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:07.192 INFO:teuthology.orchestra.run.vm06.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:07.192 INFO:teuthology.orchestra.run.vm06.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:07.192 INFO:teuthology.orchestra.run.vm06.stdout: python3-websocket python3-zc.lockfile sg3-utils sg3-utils-udev smartmontools 2026-03-24T02:54:07.192 INFO:teuthology.orchestra.run.vm06.stdout: socat xmlstarlet 2026-03-24T02:54:07.192 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:07.208 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:07.208 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:07.241 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:54:07.390 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:54:07.391 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:54:07.502 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:07.502 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:07.502 INFO:teuthology.orchestra.run.vm06.stdout: libboost-thread1.74.0 libcephfs-proxy2 libdouble-conversion3 libfuse2 2026-03-24T02:54:07.502 INFO:teuthology.orchestra.run.vm06.stdout: libgfapi0 libgfrpc0 libgfxdr0 libglusterfs0 libiscsi7 libjq1 liblttng-ust1 2026-03-24T02:54:07.502 INFO:teuthology.orchestra.run.vm06.stdout: libnbd0 liboath0 libonig5 libpcre2-16-0 libpmemobj1 libqt5core5a libqt5dbus5 2026-03-24T02:54:07.502 INFO:teuthology.orchestra.run.vm06.stdout: libqt5network5 librdkafka1 libsgutils2-2 libthrift-0.16.0 nvme-cli 2026-03-24T02:54:07.503 INFO:teuthology.orchestra.run.vm06.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:07.503 INFO:teuthology.orchestra.run.vm06.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:07.503 INFO:teuthology.orchestra.run.vm06.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:07.503 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:07.503 INFO:teuthology.orchestra.run.vm06.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:07.503 INFO:teuthology.orchestra.run.vm06.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:07.503 INFO:teuthology.orchestra.run.vm06.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:07.503 INFO:teuthology.orchestra.run.vm06.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:07.503 INFO:teuthology.orchestra.run.vm06.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n sg3-utils 2026-03-24T02:54:07.503 INFO:teuthology.orchestra.run.vm06.stdout: sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:54:07.503 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:07.512 INFO:teuthology.orchestra.run.vm06.stdout:The following packages will be REMOVED: 2026-03-24T02:54:07.512 INFO:teuthology.orchestra.run.vm06.stdout: librados2* libradosstriper1* librbd1* librgw2* libsqlite3-mod-ceph* 2026-03-24T02:54:07.512 INFO:teuthology.orchestra.run.vm06.stdout: qemu-block-extra* rbd-fuse* 2026-03-24T02:54:07.519 INFO:teuthology.orchestra.run.vm07.stdout:Removing librbd1 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:07.532 INFO:teuthology.orchestra.run.vm07.stdout:Removing librgw2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:07.543 INFO:teuthology.orchestra.run.vm07.stdout:Removing librados2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:07.567 INFO:teuthology.orchestra.run.vm07.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-03-24T02:54:07.599 INFO:teuthology.orchestra.run.vm07.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-03-24T02:54:07.665 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123650 files and directories currently installed.) 2026-03-24T02:54:07.666 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 7 to remove and 44 not upgraded. 2026-03-24T02:54:07.667 INFO:teuthology.orchestra.run.vm06.stdout:After this operation, 59.2 MB disk space will be freed. 2026-03-24T02:54:07.667 INFO:teuthology.orchestra.run.vm07.stdout:Purging configuration files for qemu-block-extra (1:6.2+dfsg-2ubuntu6.28) ... 2026-03-24T02:54:07.701 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123701 files and directories currently installed.) 2026-03-24T02:54:07.703 INFO:teuthology.orchestra.run.vm06.stdout:Removing rbd-fuse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:07.714 INFO:teuthology.orchestra.run.vm06.stdout:Removing libsqlite3-mod-ceph (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:07.725 INFO:teuthology.orchestra.run.vm06.stdout:Removing libradosstriper1 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:07.735 INFO:teuthology.orchestra.run.vm06.stdout:Removing qemu-block-extra (1:6.2+dfsg-2ubuntu6.28) ... 2026-03-24T02:54:08.118 INFO:teuthology.orchestra.run.vm06.stdout:Removing librbd1 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:08.130 INFO:teuthology.orchestra.run.vm06.stdout:Removing librgw2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:08.142 INFO:teuthology.orchestra.run.vm06.stdout:Removing librados2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:08.166 INFO:teuthology.orchestra.run.vm06.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-03-24T02:54:08.199 INFO:teuthology.orchestra.run.vm06.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-03-24T02:54:08.265 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123650 files and directories currently installed.) 2026-03-24T02:54:08.267 INFO:teuthology.orchestra.run.vm06.stdout:Purging configuration files for qemu-block-extra (1:6.2+dfsg-2ubuntu6.28) ... 2026-03-24T02:54:09.114 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:09.151 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:54:09.326 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:54:09.326 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout:Package 'librbd1' is not installed, so not removed 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout: libboost-thread1.74.0 libcephfs-proxy2 libdouble-conversion3 libfuse2 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout: libgfapi0 libgfrpc0 libgfxdr0 libglusterfs0 libiscsi7 libjq1 liblttng-ust1 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout: libnbd0 liboath0 libonig5 libpcre2-16-0 libpmemobj1 libqt5core5a libqt5dbus5 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout: libqt5network5 librdkafka1 libsgutils2-2 libthrift-0.16.0 nvme-cli 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n sg3-utils 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout: sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:54:09.438 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:09.455 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:09.455 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:09.489 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:54:09.643 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:54:09.644 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:54:09.691 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:09.726 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout:Package 'rbd-fuse' is not installed, so not removed 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout: libboost-thread1.74.0 libcephfs-proxy2 libdouble-conversion3 libfuse2 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout: libgfapi0 libgfrpc0 libgfxdr0 libglusterfs0 libiscsi7 libjq1 liblttng-ust1 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout: libnbd0 liboath0 libonig5 libpcre2-16-0 libpmemobj1 libqt5core5a libqt5dbus5 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout: libqt5network5 librdkafka1 libsgutils2-2 libthrift-0.16.0 nvme-cli 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n sg3-utils 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout: sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:54:09.755 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:09.771 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:09.771 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:09.805 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:54:09.897 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:54:09.897 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:54:09.976 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:54:09.977 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:54:10.026 INFO:teuthology.orchestra.run.vm06.stdout:Package 'librbd1' is not installed, so not removed 2026-03-24T02:54:10.026 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:10.026 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:10.026 INFO:teuthology.orchestra.run.vm06.stdout: libboost-thread1.74.0 libcephfs-proxy2 libdouble-conversion3 libfuse2 2026-03-24T02:54:10.026 INFO:teuthology.orchestra.run.vm06.stdout: libgfapi0 libgfrpc0 libgfxdr0 libglusterfs0 libiscsi7 libjq1 liblttng-ust1 2026-03-24T02:54:10.026 INFO:teuthology.orchestra.run.vm06.stdout: libnbd0 liboath0 libonig5 libpcre2-16-0 libpmemobj1 libqt5core5a libqt5dbus5 2026-03-24T02:54:10.027 INFO:teuthology.orchestra.run.vm06.stdout: libqt5network5 librdkafka1 libsgutils2-2 libthrift-0.16.0 nvme-cli 2026-03-24T02:54:10.027 INFO:teuthology.orchestra.run.vm06.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:10.027 INFO:teuthology.orchestra.run.vm06.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:10.027 INFO:teuthology.orchestra.run.vm06.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:10.027 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:10.027 INFO:teuthology.orchestra.run.vm06.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:10.027 INFO:teuthology.orchestra.run.vm06.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:10.027 INFO:teuthology.orchestra.run.vm06.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:10.027 INFO:teuthology.orchestra.run.vm06.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:10.027 INFO:teuthology.orchestra.run.vm06.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n sg3-utils 2026-03-24T02:54:10.027 INFO:teuthology.orchestra.run.vm06.stdout: sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:54:10.027 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:10.047 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:10.047 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:10.081 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:54:10.117 INFO:teuthology.orchestra.run.vm07.stdout:Package 'rbd-mirror' is not installed, so not removed 2026-03-24T02:54:10.117 INFO:teuthology.orchestra.run.vm07.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:10.117 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:10.117 INFO:teuthology.orchestra.run.vm07.stdout: libboost-thread1.74.0 libcephfs-proxy2 libdouble-conversion3 libfuse2 2026-03-24T02:54:10.117 INFO:teuthology.orchestra.run.vm07.stdout: libgfapi0 libgfrpc0 libgfxdr0 libglusterfs0 libiscsi7 libjq1 liblttng-ust1 2026-03-24T02:54:10.118 INFO:teuthology.orchestra.run.vm07.stdout: libnbd0 liboath0 libonig5 libpcre2-16-0 libpmemobj1 libqt5core5a libqt5dbus5 2026-03-24T02:54:10.118 INFO:teuthology.orchestra.run.vm07.stdout: libqt5network5 librdkafka1 libsgutils2-2 libthrift-0.16.0 nvme-cli 2026-03-24T02:54:10.118 INFO:teuthology.orchestra.run.vm07.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:10.118 INFO:teuthology.orchestra.run.vm07.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:10.118 INFO:teuthology.orchestra.run.vm07.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:10.118 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:10.118 INFO:teuthology.orchestra.run.vm07.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:10.118 INFO:teuthology.orchestra.run.vm07.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:10.118 INFO:teuthology.orchestra.run.vm07.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:10.118 INFO:teuthology.orchestra.run.vm07.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:10.118 INFO:teuthology.orchestra.run.vm07.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n sg3-utils 2026-03-24T02:54:10.118 INFO:teuthology.orchestra.run.vm07.stdout: sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:54:10.118 INFO:teuthology.orchestra.run.vm07.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:10.136 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:10.136 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:10.138 DEBUG:teuthology.orchestra.run.vm07:> dpkg -l | grep '^.\(U\|H\)R' | awk '{print $2}' | sudo xargs --no-run-if-empty dpkg -P --force-remove-reinstreq 2026-03-24T02:54:10.196 DEBUG:teuthology.orchestra.run.vm07:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" autoremove 2026-03-24T02:54:10.258 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:54:10.258 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:54:10.274 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:54:10.400 INFO:teuthology.orchestra.run.vm06.stdout:Package 'rbd-fuse' is not installed, so not removed 2026-03-24T02:54:10.400 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:10.400 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:10.400 INFO:teuthology.orchestra.run.vm06.stdout: libboost-thread1.74.0 libcephfs-proxy2 libdouble-conversion3 libfuse2 2026-03-24T02:54:10.400 INFO:teuthology.orchestra.run.vm06.stdout: libgfapi0 libgfrpc0 libgfxdr0 libglusterfs0 libiscsi7 libjq1 liblttng-ust1 2026-03-24T02:54:10.400 INFO:teuthology.orchestra.run.vm06.stdout: libnbd0 liboath0 libonig5 libpcre2-16-0 libpmemobj1 libqt5core5a libqt5dbus5 2026-03-24T02:54:10.400 INFO:teuthology.orchestra.run.vm06.stdout: libqt5network5 librdkafka1 libsgutils2-2 libthrift-0.16.0 nvme-cli 2026-03-24T02:54:10.400 INFO:teuthology.orchestra.run.vm06.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:10.400 INFO:teuthology.orchestra.run.vm06.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:10.400 INFO:teuthology.orchestra.run.vm06.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:10.400 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:10.401 INFO:teuthology.orchestra.run.vm06.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:10.401 INFO:teuthology.orchestra.run.vm06.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:10.401 INFO:teuthology.orchestra.run.vm06.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:10.401 INFO:teuthology.orchestra.run.vm06.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:10.401 INFO:teuthology.orchestra.run.vm06.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n sg3-utils 2026-03-24T02:54:10.401 INFO:teuthology.orchestra.run.vm06.stdout: sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:54:10.401 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:10.421 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:10.421 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:10.456 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:54:10.482 INFO:teuthology.orchestra.run.vm07.stdout:Building dependency tree... 2026-03-24T02:54:10.482 INFO:teuthology.orchestra.run.vm07.stdout:Reading state information... 2026-03-24T02:54:10.605 INFO:teuthology.orchestra.run.vm07.stdout:The following packages will be REMOVED: 2026-03-24T02:54:10.605 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:10.605 INFO:teuthology.orchestra.run.vm07.stdout: libboost-thread1.74.0 libcephfs-proxy2 libdouble-conversion3 libfuse2 2026-03-24T02:54:10.605 INFO:teuthology.orchestra.run.vm07.stdout: libgfapi0 libgfrpc0 libgfxdr0 libglusterfs0 libiscsi7 libjq1 liblttng-ust1 2026-03-24T02:54:10.605 INFO:teuthology.orchestra.run.vm07.stdout: libnbd0 liboath0 libonig5 libpcre2-16-0 libpmemobj1 libqt5core5a libqt5dbus5 2026-03-24T02:54:10.605 INFO:teuthology.orchestra.run.vm07.stdout: libqt5network5 librdkafka1 libsgutils2-2 libthrift-0.16.0 nvme-cli 2026-03-24T02:54:10.605 INFO:teuthology.orchestra.run.vm07.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:10.605 INFO:teuthology.orchestra.run.vm07.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:10.605 INFO:teuthology.orchestra.run.vm07.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:10.605 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:10.605 INFO:teuthology.orchestra.run.vm07.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:10.605 INFO:teuthology.orchestra.run.vm07.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:10.605 INFO:teuthology.orchestra.run.vm07.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:10.605 INFO:teuthology.orchestra.run.vm07.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:10.606 INFO:teuthology.orchestra.run.vm07.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n sg3-utils 2026-03-24T02:54:10.606 INFO:teuthology.orchestra.run.vm07.stdout: sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:54:10.646 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:54:10.646 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:54:10.773 INFO:teuthology.orchestra.run.vm07.stdout:0 upgraded, 0 newly installed, 64 to remove and 44 not upgraded. 2026-03-24T02:54:10.773 INFO:teuthology.orchestra.run.vm07.stdout:After this operation, 96.8 MB disk space will be freed. 2026-03-24T02:54:10.777 INFO:teuthology.orchestra.run.vm06.stdout:Package 'rbd-mirror' is not installed, so not removed 2026-03-24T02:54:10.777 INFO:teuthology.orchestra.run.vm06.stdout:The following packages were automatically installed and are no longer required: 2026-03-24T02:54:10.777 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:10.777 INFO:teuthology.orchestra.run.vm06.stdout: libboost-thread1.74.0 libcephfs-proxy2 libdouble-conversion3 libfuse2 2026-03-24T02:54:10.777 INFO:teuthology.orchestra.run.vm06.stdout: libgfapi0 libgfrpc0 libgfxdr0 libglusterfs0 libiscsi7 libjq1 liblttng-ust1 2026-03-24T02:54:10.777 INFO:teuthology.orchestra.run.vm06.stdout: libnbd0 liboath0 libonig5 libpcre2-16-0 libpmemobj1 libqt5core5a libqt5dbus5 2026-03-24T02:54:10.778 INFO:teuthology.orchestra.run.vm06.stdout: libqt5network5 librdkafka1 libsgutils2-2 libthrift-0.16.0 nvme-cli 2026-03-24T02:54:10.778 INFO:teuthology.orchestra.run.vm06.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:10.778 INFO:teuthology.orchestra.run.vm06.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:10.778 INFO:teuthology.orchestra.run.vm06.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:10.778 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:10.778 INFO:teuthology.orchestra.run.vm06.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:10.778 INFO:teuthology.orchestra.run.vm06.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:10.778 INFO:teuthology.orchestra.run.vm06.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:10.778 INFO:teuthology.orchestra.run.vm06.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:10.778 INFO:teuthology.orchestra.run.vm06.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n sg3-utils 2026-03-24T02:54:10.778 INFO:teuthology.orchestra.run.vm06.stdout: sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:54:10.778 INFO:teuthology.orchestra.run.vm06.stdout:Use 'sudo apt autoremove' to remove them. 2026-03-24T02:54:10.799 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. 2026-03-24T02:54:10.799 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:10.801 DEBUG:teuthology.orchestra.run.vm06:> dpkg -l | grep '^.\(U\|H\)R' | awk '{print $2}' | sudo xargs --no-run-if-empty dpkg -P --force-remove-reinstreq 2026-03-24T02:54:10.814 INFO:teuthology.orchestra.run.vm07.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123650 files and directories currently installed.) 2026-03-24T02:54:10.815 INFO:teuthology.orchestra.run.vm07.stdout:Removing ceph-mgr-modules-core (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:10.837 INFO:teuthology.orchestra.run.vm07.stdout:Removing jq (1.6-2.1ubuntu3.1) ... 2026-03-24T02:54:10.850 INFO:teuthology.orchestra.run.vm07.stdout:Removing kpartx (0.8.8-1ubuntu1.22.04.4) ... 2026-03-24T02:54:10.856 DEBUG:teuthology.orchestra.run.vm06:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" autoremove 2026-03-24T02:54:10.862 INFO:teuthology.orchestra.run.vm07.stdout:Removing libboost-iostreams1.74.0:amd64 (1.74.0-14ubuntu3) ... 2026-03-24T02:54:10.875 INFO:teuthology.orchestra.run.vm07.stdout:Removing libboost-thread1.74.0:amd64 (1.74.0-14ubuntu3) ... 2026-03-24T02:54:10.885 INFO:teuthology.orchestra.run.vm07.stdout:Removing libcephfs-proxy2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:10.895 INFO:teuthology.orchestra.run.vm07.stdout:Removing libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-03-24T02:54:10.907 INFO:teuthology.orchestra.run.vm07.stdout:Removing libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:54:10.919 INFO:teuthology.orchestra.run.vm07.stdout:Removing libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:54:10.930 INFO:teuthology.orchestra.run.vm07.stdout:Removing libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:54:10.938 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:54:10.953 INFO:teuthology.orchestra.run.vm07.stdout:Removing libdouble-conversion3:amd64 (3.1.7-4) ... 2026-03-24T02:54:10.965 INFO:teuthology.orchestra.run.vm07.stdout:Removing libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-03-24T02:54:10.979 INFO:teuthology.orchestra.run.vm07.stdout:Removing libgfapi0:amd64 (10.1-1ubuntu0.2) ... 2026-03-24T02:54:10.993 INFO:teuthology.orchestra.run.vm07.stdout:Removing libgfrpc0:amd64 (10.1-1ubuntu0.2) ... 2026-03-24T02:54:11.006 INFO:teuthology.orchestra.run.vm07.stdout:Removing libgfxdr0:amd64 (10.1-1ubuntu0.2) ... 2026-03-24T02:54:11.018 INFO:teuthology.orchestra.run.vm07.stdout:Removing libglusterfs0:amd64 (10.1-1ubuntu0.2) ... 2026-03-24T02:54:11.029 INFO:teuthology.orchestra.run.vm07.stdout:Removing libiscsi7:amd64 (1.19.0-3build2) ... 2026-03-24T02:54:11.040 INFO:teuthology.orchestra.run.vm07.stdout:Removing libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-03-24T02:54:11.051 INFO:teuthology.orchestra.run.vm07.stdout:Removing liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-03-24T02:54:11.063 INFO:teuthology.orchestra.run.vm07.stdout:Removing libnbd0 (1.10.5-1) ... 2026-03-24T02:54:11.075 INFO:teuthology.orchestra.run.vm07.stdout:Removing liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-03-24T02:54:11.087 INFO:teuthology.orchestra.run.vm07.stdout:Removing libonig5:amd64 (6.9.7.1-2build1) ... 2026-03-24T02:54:11.098 INFO:teuthology.orchestra.run.vm07.stdout:Removing libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-03-24T02:54:11.110 INFO:teuthology.orchestra.run.vm07.stdout:Removing libpmemobj1:amd64 (1.11.1-3build1) ... 2026-03-24T02:54:11.121 INFO:teuthology.orchestra.run.vm07.stdout:Removing librdkafka1:amd64 (1.8.0-1build1) ... 2026-03-24T02:54:11.131 INFO:teuthology.orchestra.run.vm07.stdout:Removing sg3-utils-udev (1.46-1ubuntu0.22.04.1) ... 2026-03-24T02:54:11.137 INFO:teuthology.orchestra.run.vm06.stdout:Building dependency tree... 2026-03-24T02:54:11.137 INFO:teuthology.orchestra.run.vm06.stdout:Reading state information... 2026-03-24T02:54:11.141 INFO:teuthology.orchestra.run.vm07.stdout:update-initramfs: deferring update (trigger activated) 2026-03-24T02:54:11.150 INFO:teuthology.orchestra.run.vm07.stdout:Removing sg3-utils (1.46-1ubuntu0.22.04.1) ... 2026-03-24T02:54:11.165 INFO:teuthology.orchestra.run.vm07.stdout:Removing libsgutils2-2:amd64 (1.46-1ubuntu0.22.04.1) ... 2026-03-24T02:54:11.174 INFO:teuthology.orchestra.run.vm07.stdout:Removing nvme-cli (1.16-3ubuntu0.3) ... 2026-03-24T02:54:11.259 INFO:teuthology.orchestra.run.vm06.stdout:The following packages will be REMOVED: 2026-03-24T02:54:11.259 INFO:teuthology.orchestra.run.vm06.stdout: ceph-mgr-modules-core jq kpartx libboost-iostreams1.74.0 2026-03-24T02:54:11.259 INFO:teuthology.orchestra.run.vm06.stdout: libboost-thread1.74.0 libcephfs-proxy2 libdouble-conversion3 libfuse2 2026-03-24T02:54:11.259 INFO:teuthology.orchestra.run.vm06.stdout: libgfapi0 libgfrpc0 libgfxdr0 libglusterfs0 libiscsi7 libjq1 liblttng-ust1 2026-03-24T02:54:11.259 INFO:teuthology.orchestra.run.vm06.stdout: libnbd0 liboath0 libonig5 libpcre2-16-0 libpmemobj1 libqt5core5a libqt5dbus5 2026-03-24T02:54:11.259 INFO:teuthology.orchestra.run.vm06.stdout: libqt5network5 librdkafka1 libsgutils2-2 libthrift-0.16.0 nvme-cli 2026-03-24T02:54:11.259 INFO:teuthology.orchestra.run.vm06.stdout: python-asyncssh-doc python3-asyncssh python3-cachetools 2026-03-24T02:54:11.259 INFO:teuthology.orchestra.run.vm06.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-03-24T02:54:11.259 INFO:teuthology.orchestra.run.vm06.stdout: python3-google-auth python3-jaraco.classes python3-jaraco.collections 2026-03-24T02:54:11.259 INFO:teuthology.orchestra.run.vm06.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-03-24T02:54:11.259 INFO:teuthology.orchestra.run.vm06.stdout: python3-kubernetes python3-natsort python3-portend python3-prettytable 2026-03-24T02:54:11.259 INFO:teuthology.orchestra.run.vm06.stdout: python3-psutil python3-repoze.lru python3-requests-oauthlib python3-routes 2026-03-24T02:54:11.259 INFO:teuthology.orchestra.run.vm06.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-03-24T02:54:11.260 INFO:teuthology.orchestra.run.vm06.stdout: python3-tempora python3-threadpoolctl python3-wcwidth python3-webob 2026-03-24T02:54:11.260 INFO:teuthology.orchestra.run.vm06.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n sg3-utils 2026-03-24T02:54:11.260 INFO:teuthology.orchestra.run.vm06.stdout: sg3-utils-udev smartmontools socat xmlstarlet 2026-03-24T02:54:11.423 INFO:teuthology.orchestra.run.vm06.stdout:0 upgraded, 0 newly installed, 64 to remove and 44 not upgraded. 2026-03-24T02:54:11.423 INFO:teuthology.orchestra.run.vm06.stdout:After this operation, 96.8 MB disk space will be freed. 2026-03-24T02:54:11.463 INFO:teuthology.orchestra.run.vm06.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 123650 files and directories currently installed.) 2026-03-24T02:54:11.465 INFO:teuthology.orchestra.run.vm06.stdout:Removing ceph-mgr-modules-core (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:11.481 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/volumes/fs/operations/versions' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/test_orchestrator' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/telemetry' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/telegraf' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/status' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/stats/fs' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/snap_schedule/fs' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/selftest' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/rgw' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/rbd_support' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/prometheus' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/progress' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/pg_autoscaler' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/osd_support' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/osd_perf_query' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/orchestrator' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/nfs' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/mirroring/fs/dir_map' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/localpool' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/iostat' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/insights' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/influx' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/devicehealth' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/crash' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/balancer' not empty so not removed 2026-03-24T02:54:11.482 INFO:teuthology.orchestra.run.vm06.stdout:dpkg: warning: while removing ceph-mgr-modules-core, directory '/usr/share/ceph/mgr/alerts' not empty so not removed 2026-03-24T02:54:11.488 INFO:teuthology.orchestra.run.vm06.stdout:Removing jq (1.6-2.1ubuntu3.1) ... 2026-03-24T02:54:11.500 INFO:teuthology.orchestra.run.vm06.stdout:Removing kpartx (0.8.8-1ubuntu1.22.04.4) ... 2026-03-24T02:54:11.511 INFO:teuthology.orchestra.run.vm06.stdout:Removing libboost-iostreams1.74.0:amd64 (1.74.0-14ubuntu3) ... 2026-03-24T02:54:11.523 INFO:teuthology.orchestra.run.vm06.stdout:Removing libboost-thread1.74.0:amd64 (1.74.0-14ubuntu3) ... 2026-03-24T02:54:11.534 INFO:teuthology.orchestra.run.vm06.stdout:Removing libcephfs-proxy2 (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:11.544 INFO:teuthology.orchestra.run.vm06.stdout:Removing libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-03-24T02:54:11.555 INFO:teuthology.orchestra.run.vm06.stdout:Removing libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:54:11.567 INFO:teuthology.orchestra.run.vm06.stdout:Removing libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:54:11.578 INFO:teuthology.orchestra.run.vm06.stdout:Removing libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-03-24T02:54:11.594 INFO:teuthology.orchestra.run.vm07.stdout:Removing python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-03-24T02:54:11.599 INFO:teuthology.orchestra.run.vm06.stdout:Removing libdouble-conversion3:amd64 (3.1.7-4) ... 2026-03-24T02:54:11.609 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-03-24T02:54:11.610 INFO:teuthology.orchestra.run.vm06.stdout:Removing libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-03-24T02:54:11.622 INFO:teuthology.orchestra.run.vm06.stdout:Removing libgfapi0:amd64 (10.1-1ubuntu0.2) ... 2026-03-24T02:54:11.633 INFO:teuthology.orchestra.run.vm06.stdout:Removing libgfrpc0:amd64 (10.1-1ubuntu0.2) ... 2026-03-24T02:54:11.644 INFO:teuthology.orchestra.run.vm06.stdout:Removing libgfxdr0:amd64 (10.1-1ubuntu0.2) ... 2026-03-24T02:54:11.655 INFO:teuthology.orchestra.run.vm06.stdout:Removing libglusterfs0:amd64 (10.1-1ubuntu0.2) ... 2026-03-24T02:54:11.666 INFO:teuthology.orchestra.run.vm06.stdout:Removing libiscsi7:amd64 (1.19.0-3build2) ... 2026-03-24T02:54:11.669 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-kubernetes (12.0.1-1ubuntu1) ... 2026-03-24T02:54:11.678 INFO:teuthology.orchestra.run.vm06.stdout:Removing libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-03-24T02:54:11.689 INFO:teuthology.orchestra.run.vm06.stdout:Removing liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-03-24T02:54:11.700 INFO:teuthology.orchestra.run.vm06.stdout:Removing libnbd0 (1.10.5-1) ... 2026-03-24T02:54:11.710 INFO:teuthology.orchestra.run.vm06.stdout:Removing liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-03-24T02:54:11.725 INFO:teuthology.orchestra.run.vm06.stdout:Removing libonig5:amd64 (6.9.7.1-2build1) ... 2026-03-24T02:54:11.736 INFO:teuthology.orchestra.run.vm06.stdout:Removing libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-03-24T02:54:11.746 INFO:teuthology.orchestra.run.vm06.stdout:Removing libpmemobj1:amd64 (1.11.1-3build1) ... 2026-03-24T02:54:11.758 INFO:teuthology.orchestra.run.vm06.stdout:Removing librdkafka1:amd64 (1.8.0-1build1) ... 2026-03-24T02:54:11.770 INFO:teuthology.orchestra.run.vm06.stdout:Removing sg3-utils-udev (1.46-1ubuntu0.22.04.1) ... 2026-03-24T02:54:11.778 INFO:teuthology.orchestra.run.vm06.stdout:update-initramfs: deferring update (trigger activated) 2026-03-24T02:54:11.789 INFO:teuthology.orchestra.run.vm06.stdout:Removing sg3-utils (1.46-1ubuntu0.22.04.1) ... 2026-03-24T02:54:11.805 INFO:teuthology.orchestra.run.vm06.stdout:Removing libsgutils2-2:amd64 (1.46-1ubuntu0.22.04.1) ... 2026-03-24T02:54:11.816 INFO:teuthology.orchestra.run.vm06.stdout:Removing nvme-cli (1.16-3ubuntu0.3) ... 2026-03-24T02:54:11.944 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-google-auth (1.5.1-3) ... 2026-03-24T02:54:12.012 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-cachetools (5.0.0-1) ... 2026-03-24T02:54:12.067 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-ceph-argparse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:12.118 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-ceph-common (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:12.177 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-cherrypy3 (18.6.1-4) ... 2026-03-24T02:54:12.236 INFO:teuthology.orchestra.run.vm06.stdout:Removing python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-03-24T02:54:12.249 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-03-24T02:54:12.251 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-03-24T02:54:12.302 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-jaraco.collections (3.4.0-2) ... 2026-03-24T02:54:12.310 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-kubernetes (12.0.1-1ubuntu1) ... 2026-03-24T02:54:12.351 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-jaraco.classes (3.2.1-3) ... 2026-03-24T02:54:12.405 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-portend (3.0.0-1) ... 2026-03-24T02:54:12.455 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-tempora (4.1.2-1) ... 2026-03-24T02:54:12.504 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-jaraco.text (3.6.0-2) ... 2026-03-24T02:54:12.553 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-jaraco.functools (3.4.0-2) ... 2026-03-24T02:54:12.602 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-sklearn (0.23.2-5ubuntu6) ... 2026-03-24T02:54:12.610 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-google-auth (1.5.1-3) ... 2026-03-24T02:54:12.669 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-cachetools (5.0.0-1) ... 2026-03-24T02:54:12.724 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-ceph-argparse (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:12.740 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-joblib (0.17.0-4ubuntu1) ... 2026-03-24T02:54:12.777 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-ceph-common (20.2.0-712-g70f8415b-1jammy) ... 2026-03-24T02:54:12.804 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-natsort (8.0.2-1) ... 2026-03-24T02:54:12.833 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-cherrypy3 (18.6.1-4) ... 2026-03-24T02:54:12.857 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-prettytable (2.5.0-2) ... 2026-03-24T02:54:12.906 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-03-24T02:54:12.911 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-psutil (5.9.0-1build1) ... 2026-03-24T02:54:12.964 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-jaraco.collections (3.4.0-2) ... 2026-03-24T02:54:12.967 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-routes (2.5.1-1ubuntu1) ... 2026-03-24T02:54:13.021 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-jaraco.classes (3.2.1-3) ... 2026-03-24T02:54:13.022 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-repoze.lru (0.7-2) ... 2026-03-24T02:54:13.073 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-03-24T02:54:13.077 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-portend (3.0.0-1) ... 2026-03-24T02:54:13.127 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-rsa (4.8-1) ... 2026-03-24T02:54:13.129 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-tempora (4.1.2-1) ... 2026-03-24T02:54:13.179 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-simplejson (3.17.6-1build1) ... 2026-03-24T02:54:13.185 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-jaraco.text (3.6.0-2) ... 2026-03-24T02:54:13.234 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-03-24T02:54:13.237 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-jaraco.functools (3.4.0-2) ... 2026-03-24T02:54:13.247 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-threadpoolctl (3.1.0-1) ... 2026-03-24T02:54:13.290 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-sklearn (0.23.2-5ubuntu6) ... 2026-03-24T02:54:13.299 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-wcwidth (0.2.5+dfsg1-1) ... 2026-03-24T02:54:13.358 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-03-24T02:54:13.415 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-websocket (1.2.3-1) ... 2026-03-24T02:54:13.432 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-joblib (0.17.0-4ubuntu1) ... 2026-03-24T02:54:13.473 INFO:teuthology.orchestra.run.vm07.stdout:Removing python3-zc.lockfile (2.0-1) ... 2026-03-24T02:54:13.491 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-natsort (8.0.2-1) ... 2026-03-24T02:54:13.523 INFO:teuthology.orchestra.run.vm07.stdout:Removing qttranslations5-l10n (5.15.3-1) ... 2026-03-24T02:54:13.542 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-prettytable (2.5.0-2) ... 2026-03-24T02:54:13.544 INFO:teuthology.orchestra.run.vm07.stdout:Removing smartmontools (7.2-1ubuntu0.1) ... 2026-03-24T02:54:13.593 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-psutil (5.9.0-1build1) ... 2026-03-24T02:54:13.642 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-routes (2.5.1-1ubuntu1) ... 2026-03-24T02:54:13.692 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-repoze.lru (0.7-2) ... 2026-03-24T02:54:13.739 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-03-24T02:54:13.786 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-rsa (4.8-1) ... 2026-03-24T02:54:13.835 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-simplejson (3.17.6-1build1) ... 2026-03-24T02:54:13.887 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-03-24T02:54:13.900 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-threadpoolctl (3.1.0-1) ... 2026-03-24T02:54:13.907 INFO:teuthology.orchestra.run.vm07.stdout:Removing socat (1.7.4.1-3ubuntu4) ... 2026-03-24T02:54:13.920 INFO:teuthology.orchestra.run.vm07.stdout:Removing xmlstarlet (1.6.1-2.1) ... 2026-03-24T02:54:13.949 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-wcwidth (0.2.5+dfsg1-1) ... 2026-03-24T02:54:13.952 INFO:teuthology.orchestra.run.vm07.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-03-24T02:54:13.963 INFO:teuthology.orchestra.run.vm07.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-03-24T02:54:13.998 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-03-24T02:54:14.013 INFO:teuthology.orchestra.run.vm07.stdout:Processing triggers for initramfs-tools (0.140ubuntu13.5) ... 2026-03-24T02:54:14.030 INFO:teuthology.orchestra.run.vm07.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-171-generic 2026-03-24T02:54:14.050 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-websocket (1.2.3-1) ... 2026-03-24T02:54:14.102 INFO:teuthology.orchestra.run.vm06.stdout:Removing python3-zc.lockfile (2.0-1) ... 2026-03-24T02:54:14.150 INFO:teuthology.orchestra.run.vm06.stdout:Removing qttranslations5-l10n (5.15.3-1) ... 2026-03-24T02:54:14.168 INFO:teuthology.orchestra.run.vm06.stdout:Removing smartmontools (7.2-1ubuntu0.1) ... 2026-03-24T02:54:14.577 INFO:teuthology.orchestra.run.vm06.stdout:Removing socat (1.7.4.1-3ubuntu4) ... 2026-03-24T02:54:14.588 INFO:teuthology.orchestra.run.vm06.stdout:Removing xmlstarlet (1.6.1-2.1) ... 2026-03-24T02:54:14.619 INFO:teuthology.orchestra.run.vm06.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-03-24T02:54:14.630 INFO:teuthology.orchestra.run.vm06.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-03-24T02:54:14.679 INFO:teuthology.orchestra.run.vm06.stdout:Processing triggers for initramfs-tools (0.140ubuntu13.5) ... 2026-03-24T02:54:14.695 INFO:teuthology.orchestra.run.vm06.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-171-generic 2026-03-24T02:54:18.762 INFO:teuthology.orchestra.run.vm07.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:18.765 DEBUG:teuthology.parallel:result is None 2026-03-24T02:54:19.185 INFO:teuthology.orchestra.run.vm06.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-03-24T02:54:19.188 DEBUG:teuthology.parallel:result is None 2026-03-24T02:54:19.188 INFO:teuthology.task.install:Removing ceph sources lists on ubuntu@vm06.local 2026-03-24T02:54:19.188 INFO:teuthology.task.install:Removing ceph sources lists on ubuntu@vm07.local 2026-03-24T02:54:19.191 DEBUG:teuthology.orchestra.run.vm06:> sudo rm -f /etc/apt/sources.list.d/ceph.list 2026-03-24T02:54:19.191 DEBUG:teuthology.orchestra.run.vm07:> sudo rm -f /etc/apt/sources.list.d/ceph.list 2026-03-24T02:54:19.199 DEBUG:teuthology.orchestra.run.vm07:> sudo apt-get update 2026-03-24T02:54:19.238 DEBUG:teuthology.orchestra.run.vm06:> sudo apt-get update 2026-03-24T02:54:19.364 INFO:teuthology.orchestra.run.vm07.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2026-03-24T02:54:19.394 INFO:teuthology.orchestra.run.vm07.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2026-03-24T02:54:19.402 INFO:teuthology.orchestra.run.vm06.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2026-03-24T02:54:19.428 INFO:teuthology.orchestra.run.vm07.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2026-03-24T02:54:19.432 INFO:teuthology.orchestra.run.vm06.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2026-03-24T02:54:19.467 INFO:teuthology.orchestra.run.vm06.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2026-03-24T02:54:19.492 INFO:teuthology.orchestra.run.vm07.stdout:Hit:4 http://security.ubuntu.com/ubuntu jammy-security InRelease 2026-03-24T02:54:19.521 INFO:teuthology.orchestra.run.vm06.stdout:Hit:4 http://security.ubuntu.com/ubuntu jammy-security InRelease 2026-03-24T02:54:20.366 INFO:teuthology.orchestra.run.vm07.stdout:Reading package lists... 2026-03-24T02:54:20.378 DEBUG:teuthology.parallel:result is None 2026-03-24T02:54:20.523 INFO:teuthology.orchestra.run.vm06.stdout:Reading package lists... 2026-03-24T02:54:20.535 DEBUG:teuthology.parallel:result is None 2026-03-24T02:54:20.536 DEBUG:teuthology.run_tasks:Unwinding manager clock 2026-03-24T02:54:20.539 INFO:teuthology.task.clock:Checking final clock skew... 2026-03-24T02:54:20.539 DEBUG:teuthology.orchestra.run.vm06:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-03-24T02:54:20.540 DEBUG:teuthology.orchestra.run.vm07:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm07.stdout: remote refid st t when poll reach delay offset jitter 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm07.stdout:============================================================================== 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm07.stdout: 0.ubuntu.pool.n .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm07.stdout: 1.ubuntu.pool.n .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm07.stdout: 2.ubuntu.pool.n .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm07.stdout: 3.ubuntu.pool.n .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm07.stdout: ntp.ubuntu.com .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm07.stdout:-time.cloudflare 10.163.8.4 3 u 18 256 377 20.403 +2.457 0.211 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm07.stdout:*static.222.16.4 35.73.197.144 2 u 35 128 377 0.500 +0.963 0.349 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm07.stdout:-mail.anyvm.tech 144.76.140.114 3 u 22 128 377 23.575 +1.242 0.146 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm07.stdout:+mail.gunnarhofm 192.53.103.108 2 u 75 128 377 25.040 +1.161 0.135 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm07.stdout:+141.144.230.32 192.53.103.108 2 u 69 128 377 20.978 +1.264 0.554 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm06.stdout: remote refid st t when poll reach delay offset jitter 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm06.stdout:============================================================================== 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm06.stdout: 0.ubuntu.pool.n .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm06.stdout: 1.ubuntu.pool.n .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm06.stdout: 2.ubuntu.pool.n .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm06.stdout: 3.ubuntu.pool.n .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm06.stdout: ntp.ubuntu.com .POOL. 16 p - 64 0 0.000 +0.000 0.000 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm06.stdout:-time.cloudflare 10.160.8.4 3 u 33 256 377 20.370 +2.873 0.507 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm06.stdout:-mail.anyvm.tech 144.76.140.114 3 u 21 128 377 23.522 +1.206 0.488 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm06.stdout:+mail.gunnarhofm 192.53.103.108 2 u 27 128 377 25.060 +1.457 0.371 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm06.stdout:+141.144.230.32 192.53.103.108 2 u 26 128 377 20.954 +1.188 0.435 2026-03-24T02:54:20.571 INFO:teuthology.orchestra.run.vm06.stdout:*static.222.16.4 35.73.197.144 2 u 21 128 377 0.370 +2.010 0.756 2026-03-24T02:54:20.572 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2026-03-24T02:54:20.575 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2026-03-24T02:54:20.578 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2026-03-24T02:54:20.582 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2026-03-24T02:54:20.585 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2026-03-24T02:54:20.588 INFO:teuthology.task.internal:Duration was 2667.667450 seconds 2026-03-24T02:54:20.592 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2026-03-24T02:54:20.595 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2026-03-24T02:54:20.595 DEBUG:teuthology.orchestra.run.vm06:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-03-24T02:54:20.615 DEBUG:teuthology.orchestra.run.vm07:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-03-24T02:54:20.637 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2026-03-24T02:54:20.637 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@vm06.local 2026-03-24T02:54:20.637 DEBUG:teuthology.orchestra.run.vm06:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-03-24T02:54:20.685 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@vm07.local 2026-03-24T02:54:20.685 DEBUG:teuthology.orchestra.run.vm07:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-03-24T02:54:20.692 INFO:teuthology.task.internal.syslog:Gathering journactl... 2026-03-24T02:54:20.692 DEBUG:teuthology.orchestra.run.vm06:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-03-24T02:54:20.727 DEBUG:teuthology.orchestra.run.vm07:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-03-24T02:54:20.755 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2026-03-24T02:54:20.755 DEBUG:teuthology.orchestra.run.vm06:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-03-24T02:54:20.791 DEBUG:teuthology.orchestra.run.vm07:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-03-24T02:54:20.796 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-03-24T02:54:20.797 INFO:teuthology.orchestra.run.vm06.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-03-24T02:54:20.797 INFO:teuthology.orchestra.run.vm06.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-03-24T02:54:20.797 INFO:teuthology.orchestra.run.vm06.stderr: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-03-24T02:54:20.798 INFO:teuthology.orchestra.run.vm06.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-03-24T02:54:20.801 INFO:teuthology.orchestra.run.vm06.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 86.2% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-03-24T02:54:20.805 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-03-24T02:54:20.805 INFO:teuthology.orchestra.run.vm07.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-03-24T02:54:20.805 INFO:teuthology.orchestra.run.vm07.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-03-24T02:54:20.806 INFO:teuthology.orchestra.run.vm07.stderr: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-03-24T02:54:20.806 INFO:teuthology.orchestra.run.vm07.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-03-24T02:54:20.809 INFO:teuthology.orchestra.run.vm07.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 86.5% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-03-24T02:54:20.810 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2026-03-24T02:54:20.814 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2026-03-24T02:54:20.814 DEBUG:teuthology.orchestra.run.vm06:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-03-24T02:54:20.851 DEBUG:teuthology.orchestra.run.vm07:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-03-24T02:54:20.862 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2026-03-24T02:54:20.866 DEBUG:teuthology.orchestra.run.vm06:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-03-24T02:54:20.895 DEBUG:teuthology.orchestra.run.vm07:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-03-24T02:54:20.902 INFO:teuthology.orchestra.run.vm06.stdout:kernel.core_pattern = core 2026-03-24T02:54:20.909 INFO:teuthology.orchestra.run.vm07.stdout:kernel.core_pattern = core 2026-03-24T02:54:20.916 DEBUG:teuthology.orchestra.run.vm06:> test -e /home/ubuntu/cephtest/archive/coredump 2026-03-24T02:54:20.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-24T02:54:20.953 DEBUG:teuthology.orchestra.run.vm07:> test -e /home/ubuntu/cephtest/archive/coredump 2026-03-24T02:54:20.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-24T02:54:20.960 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2026-03-24T02:54:20.964 INFO:teuthology.task.internal:Transferring archived files... 2026-03-24T02:54:20.968 DEBUG:teuthology.misc:Transferring archived files from vm06:/home/ubuntu/cephtest/archive to /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3539/remote/vm06 2026-03-24T02:54:20.968 DEBUG:teuthology.orchestra.run.vm06:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-03-24T02:54:21.002 DEBUG:teuthology.misc:Transferring archived files from vm07:/home/ubuntu/cephtest/archive to /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3539/remote/vm07 2026-03-24T02:54:21.002 DEBUG:teuthology.orchestra.run.vm07:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-03-24T02:54:21.009 INFO:teuthology.task.internal:Removing archive directory... 2026-03-24T02:54:21.009 DEBUG:teuthology.orchestra.run.vm06:> rm -rf -- /home/ubuntu/cephtest/archive 2026-03-24T02:54:21.043 DEBUG:teuthology.orchestra.run.vm07:> rm -rf -- /home/ubuntu/cephtest/archive 2026-03-24T02:54:21.052 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2026-03-24T02:54:21.056 INFO:teuthology.task.internal:Not uploading archives. 2026-03-24T02:54:21.056 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2026-03-24T02:54:21.059 INFO:teuthology.task.internal:Tidying up after the test... 2026-03-24T02:54:21.059 DEBUG:teuthology.orchestra.run.vm06:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-03-24T02:54:21.087 DEBUG:teuthology.orchestra.run.vm07:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-03-24T02:54:21.089 INFO:teuthology.orchestra.run.vm06.stdout: 258078 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 24 02:54 /home/ubuntu/cephtest 2026-03-24T02:54:21.097 INFO:teuthology.orchestra.run.vm07.stdout: 258069 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 24 02:54 /home/ubuntu/cephtest 2026-03-24T02:54:21.097 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2026-03-24T02:54:21.106 INFO:teuthology.run:Summary data: description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-journal-stress-workunit} duration: 2667.667450428009 failure_reason: '"2026-03-24T02:47:32.483601+0000 mon.a (mon.0) 1455 : cluster [ERR] Health check failed: mon a is very low on available space (MON_DISK_CRIT)" in cluster log' flavor: default owner: kyr success: false 2026-03-24T02:54:21.107 DEBUG:teuthology.report:Pushing job info to http://localhost:8080 2026-03-24T02:54:21.127 INFO:teuthology.run:FAIL